1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package storage
16
17 import (
18 "context"
19 "crypto/tls"
20 "encoding/json"
21 "fmt"
22 "io"
23 "io/ioutil"
24 "log"
25 "net"
26 "net/http"
27 "net/http/httptest"
28 "net/url"
29 "os"
30 "reflect"
31 "regexp"
32 "sort"
33 "strings"
34 "testing"
35 "time"
36
37 "cloud.google.com/go/iam"
38 "cloud.google.com/go/internal/testutil"
39 "cloud.google.com/go/storage/internal/apiv2/storagepb"
40 "github.com/google/go-cmp/cmp"
41 "github.com/googleapis/gax-go/v2"
42 "google.golang.org/api/iterator"
43 "google.golang.org/api/option"
44 raw "google.golang.org/api/storage/v1"
45 "google.golang.org/protobuf/proto"
46 "google.golang.org/protobuf/types/known/timestamppb"
47 )
48
49 func TestV2HeaderSanitization(t *testing.T) {
50 t.Parallel()
51 var tests = []struct {
52 desc string
53 in []string
54 want []string
55 }{
56 {
57 desc: "already sanitized headers should not be modified",
58 in: []string{"x-goog-header1:true", "x-goog-header2:0"},
59 want: []string{"x-goog-header1:true", "x-goog-header2:0"},
60 },
61 {
62 desc: "sanitized headers should be sorted",
63 in: []string{"x-goog-header2:0", "x-goog-header1:true"},
64 want: []string{"x-goog-header1:true", "x-goog-header2:0"},
65 },
66 {
67 desc: "non-canonical headers should be removed",
68 in: []string{"x-goog-header1:true", "x-goog-no-value", "non-canonical-header:not-of-use"},
69 want: []string{"x-goog-header1:true"},
70 },
71 {
72 desc: "excluded canonical headers should be removed",
73 in: []string{"x-goog-header1:true", "x-goog-encryption-key:my_key", "x-goog-encryption-key-sha256:my_sha256"},
74 want: []string{"x-goog-header1:true"},
75 },
76 {
77 desc: "dirty headers should be formatted correctly",
78 in: []string{" x-goog-header1 : \textra-spaces ", "X-Goog-Header2:CamelCaseValue"},
79 want: []string{"x-goog-header1:extra-spaces", "x-goog-header2:CamelCaseValue"},
80 },
81 {
82 desc: "duplicate headers should be merged",
83 in: []string{"x-goog-header1:value1", "X-Goog-Header1:value2"},
84 want: []string{"x-goog-header1:value1,value2"},
85 },
86 }
87 for _, test := range tests {
88 got := v2SanitizeHeaders(test.in)
89 if !testutil.Equal(got, test.want) {
90 t.Errorf("%s: got %v, want %v", test.desc, got, test.want)
91 }
92 }
93 }
94
95 func TestV4HeaderSanitization(t *testing.T) {
96 t.Parallel()
97 var tests = []struct {
98 desc string
99 in []string
100 want []string
101 }{
102 {
103 desc: "already sanitized headers should not be modified",
104 in: []string{"x-goog-header1:true", "x-goog-header2:0"},
105 want: []string{"x-goog-header1:true", "x-goog-header2:0"},
106 },
107 {
108 desc: "dirty headers should be formatted correctly",
109 in: []string{" x-goog-header1 : \textra-spaces ", "X-Goog-Header2:CamelCaseValue"},
110 want: []string{"x-goog-header1:extra-spaces", "x-goog-header2:CamelCaseValue"},
111 },
112 {
113 desc: "duplicate headers should be merged",
114 in: []string{"x-goog-header1:value1", "X-Goog-Header1:value2"},
115 want: []string{"x-goog-header1:value1,value2"},
116 },
117 {
118 desc: "multiple spaces in value are stripped down to one",
119 in: []string{"foo:bar gaz"},
120 want: []string{"foo:bar gaz"},
121 },
122 {
123 desc: "headers with colons in value are preserved",
124 in: []string{"x-goog-meta-start-time: 2023-02-10T02:00:00Z"},
125 want: []string{"x-goog-meta-start-time:2023-02-10T02:00:00Z"},
126 },
127 {
128 desc: "headers that end in a colon in value are preserved",
129 in: []string{"x-goog-meta-start-time: 2023-02-10T02:"},
130 want: []string{"x-goog-meta-start-time:2023-02-10T02:"},
131 },
132 }
133 for _, test := range tests {
134 got := v4SanitizeHeaders(test.in)
135 sort.Strings(got)
136 sort.Strings(test.want)
137 if !testutil.Equal(got, test.want) {
138 t.Errorf("%s: got %v, want %v", test.desc, got, test.want)
139 }
140 }
141 }
142
143 func TestSignedURLV2(t *testing.T) {
144 expires, _ := time.Parse(time.RFC3339, "2002-10-02T10:00:00-05:00")
145
146 tests := []struct {
147 desc string
148 objectName string
149 opts *SignedURLOptions
150 want string
151 }{
152 {
153 desc: "SignedURLV2 works",
154 objectName: "object-name",
155 opts: &SignedURLOptions{
156 GoogleAccessID: "xxx@clientid",
157 PrivateKey: dummyKey("rsa"),
158 Method: "GET",
159 MD5: "ICy5YqxZB1uWSwcVLSNLcA==",
160 Expires: expires,
161 ContentType: "application/json",
162 Headers: []string{"x-goog-header1:true", "x-goog-header2:false"},
163 },
164 want: "https://storage.googleapis.com/bucket-name/object-name?" +
165 "Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=" +
166 "RfsHlPtbB2JUYjzCgNr2Mi%2BjggdEuL1V7E6N9o6aaqwVLBDuTv3I0%2B9" +
167 "x94E6rmmr%2FVgnmZigkIUxX%2Blfl7LgKf30uPGLt0mjKGH2p7r9ey1ONJ" +
168 "%2BhVec23FnTRcSgopglvHPuCMWU2oNJE%2F1y8EwWE27baHrG1RhRHbLVF" +
169 "bPpLZ9xTRFK20pluIkfHV00JGljB1imqQHXM%2B2XPWqBngLr%2FwqxLN7i" +
170 "FcUiqR8xQEOHF%2F2e7fbkTHPNq4TazaLZ8X0eZ3eFdJ55A5QmNi8atlN4W" +
171 "5q7Hvs0jcxElG3yqIbx439A995BkspLiAcA%2Fo4%2BxAwEMkGLICdbvakq" +
172 "3eEprNCojw%3D%3D",
173 },
174 {
175 desc: "With a PEM Private Key",
176 objectName: "object-name",
177 opts: &SignedURLOptions{
178 GoogleAccessID: "xxx@clientid",
179 PrivateKey: dummyKey("pem"),
180 Method: "GET",
181 MD5: "ICy5YqxZB1uWSwcVLSNLcA==",
182 Expires: expires,
183 ContentType: "application/json",
184 Headers: []string{"x-goog-header1:true", "x-goog-header2:false"},
185 },
186 want: "https://storage.googleapis.com/bucket-name/object-name?" +
187 "Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=" +
188 "TiyKD%2FgGb6Kh0kkb2iF%2FfF%2BnTx7L0J4YiZua8AcTmnidutePEGIU5" +
189 "NULYlrGl6l52gz4zqFb3VFfIRTcPXMdXnnFdMCDhz2QuJBUpsU1Ai9zlyTQ" +
190 "dkb6ShG03xz9%2BEXWAUQO4GBybJw%2FULASuv37xA00SwLdkqj8YdyS5II" +
191 "1lro%3D",
192 },
193 {
194 desc: "With custom SignBytes",
195 objectName: "object-name",
196 opts: &SignedURLOptions{
197 GoogleAccessID: "xxx@clientid",
198 SignBytes: func(b []byte) ([]byte, error) {
199 return []byte("signed"), nil
200 },
201 Method: "GET",
202 MD5: "ICy5YqxZB1uWSwcVLSNLcA==",
203 Expires: expires,
204 ContentType: "application/json",
205 Headers: []string{"x-goog-header1:true", "x-goog-header2:false"},
206 },
207 want: "https://storage.googleapis.com/bucket-name/object-name?" +
208 "Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=" +
209 "c2lnbmVk",
210 },
211 {
212 desc: "With unsafe object name",
213 objectName: "object name界",
214 opts: &SignedURLOptions{
215 GoogleAccessID: "xxx@clientid",
216 PrivateKey: dummyKey("pem"),
217 Method: "GET",
218 MD5: "ICy5YqxZB1uWSwcVLSNLcA==",
219 Expires: expires,
220 ContentType: "application/json",
221 Headers: []string{"x-goog-header1:true", "x-goog-header2:false"},
222 },
223 want: "https://storage.googleapis.com/bucket-name/object%20name%E7%95%8C?" +
224 "Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=bxVH1%2Bl%2" +
225 "BSxpnj3XuqKz6mOFk6M94Y%2B4w85J6FCmJan%2FNhGSpndP6fAw1uLHlOn%2F8xUaY%2F" +
226 "SfZ5GzcQ%2BbxOL1WA37yIwZ7xgLYlO%2ByAi3GuqMUmHZiNCai28emODXQ8RtWHvgv6dE" +
227 "SQ%2F0KpDMIWW7rYCaUa63UkUyeSQsKhrVqkIA%3D",
228 },
229 }
230
231 for _, test := range tests {
232 u, err := SignedURL("bucket-name", test.objectName, test.opts)
233 if err != nil {
234 t.Fatalf("[%s] %v", test.desc, err)
235 }
236 if u != test.want {
237 t.Fatalf("[%s] Unexpected signed URL; found %v", test.desc, u)
238 }
239 }
240 }
241
242 func TestSignedURLV4(t *testing.T) {
243 expires, _ := time.Parse(time.RFC3339, "2002-10-02T10:00:00-05:00")
244
245 tests := []struct {
246 desc string
247 objectName string
248 now time.Time
249 opts *SignedURLOptions
250
251
252
253
254
255 want string
256 }{
257 {
258 desc: "SignURLV4 works",
259 objectName: "object-name",
260 now: expires.Add(-24 * time.Hour),
261 opts: &SignedURLOptions{
262 GoogleAccessID: "xxx@clientid",
263 PrivateKey: dummyKey("rsa"),
264 Method: "POST",
265 Expires: expires,
266 Scheme: SigningSchemeV4,
267 ContentType: "application/json",
268 MD5: "ICy5YqxZB1uWSwcVLSNLcA==",
269 Headers: []string{"x-goog-header1:true", "x-goog-header2:false"},
270 },
271 want: "https://storage.googleapis.com/bucket-name/object-name" +
272 "?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
273 "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
274 "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
275 "&X-Goog-Signature=774b11d89663d0562b0909131b8495e70d24e31f3417d3f8fd1438a72b620b256111a7221fecab14a6ebb7dc7eed7984316a794789beb4ecdda67a77407f6de1a68113e8fa2b885e330036a995c08f0f2a7d2c212a3d0a2fd1b392d40305d3fe31ab94c547a7541278f4a956ebb6565ebe4cb27f26e30b334adb7b065adc0d27f9eaa42ee76d75d673fc4523d023d9a636de0b5329f5dffbf80024cf21fdc6236e89aa41976572bfe4807be9a9a01f644ed9f546dcf1e0394665be7610f58c36b3d63379f4d1b64f646f7427f1fc55bb89d7fdd59017d007156c99e26440e828581cddf83faf03e739e5987c062d503f2b73f24049c25edc60ecbbc09f6ce945" +
276 "&X-Goog-SignedHeaders=content-md5%3Bcontent-type%3Bhost%3Bx-goog-header1%3Bx-goog-header2",
277 },
278 {
279 desc: "With PEM Private Key",
280 objectName: "object-name",
281 now: expires.Add(-24 * time.Hour),
282 opts: &SignedURLOptions{
283 GoogleAccessID: "xxx@clientid",
284 PrivateKey: dummyKey("pem"),
285 Method: "GET",
286 Expires: expires,
287 Scheme: SigningSchemeV4,
288 },
289 want: "https://storage.googleapis.com/bucket-name/object-name" +
290 "?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
291 "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
292 "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
293 "&X-Goog-Signature=5592f4b8b2cae14025b619546d69bb463ca8f2caaab538a3cc6b5868c8c64b83a8b04b57d8a82c8696a192f62abddc8d99e0454b3fc33feac5bf87c353f0703aab6cfee60364aaeecec2edd37c1d6e6793d90812b5811b7936a014a3efad5d08477b4fbfaebf04fa61f1ca03f31bcdc46a161868cd2f4e98def6c82634a01454" +
294 "&X-Goog-SignedHeaders=host",
295 },
296 {
297 desc: "Unsafe object name",
298 objectName: "object name界",
299 now: expires.Add(-24 * time.Hour),
300 opts: &SignedURLOptions{
301 GoogleAccessID: "xxx@clientid",
302 PrivateKey: dummyKey("pem"),
303 Method: "GET",
304 Expires: expires,
305 Scheme: SigningSchemeV4,
306 },
307 want: "https://storage.googleapis.com/bucket-name/object%20name%E7%95%8C" +
308 "?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
309 "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
310 "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
311 "&X-Goog-Signature=90fd455fb47725b45c08d65ddf99078184710ad30f09bc2a190c5416ba1596e4c58420e2e48744b03de2d1b85dc8679dcb4c36af6e7a1b2547cd62becaad72aebbbaf7c1686f1aa0fedf8a9b01cef20a8b8630d824a6f8b81bb9eb75f342a7d8a28457a4efd2baac93e37089b84b1506b2af72712187f638e0eafbac650b071a" +
312 "&X-Goog-SignedHeaders=host",
313 },
314 {
315 desc: "With custom SignBytes",
316 objectName: "object-name",
317 now: expires.Add(-24 * time.Hour),
318 opts: &SignedURLOptions{
319 GoogleAccessID: "xxx@clientid",
320 SignBytes: func(b []byte) ([]byte, error) {
321 return []byte("signed"), nil
322 },
323 Method: "GET",
324 Expires: expires,
325 Scheme: SigningSchemeV4,
326 },
327 want: "https://storage.googleapis.com/bucket-name/object-name" +
328 "?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
329 "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
330 "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
331 "&X-Goog-Signature=7369676e6564" +
332 "&X-Goog-SignedHeaders=host",
333 },
334 }
335 oldUTCNow := utcNow
336 defer func() {
337 utcNow = oldUTCNow
338 }()
339
340 for _, test := range tests {
341 t.Logf("Testcase: '%s'", test.desc)
342
343 utcNow = func() time.Time {
344 return test.now
345 }
346 got, err := SignedURL("bucket-name", test.objectName, test.opts)
347 if err != nil {
348 t.Fatal(err)
349 }
350 if got != test.want {
351 t.Fatalf("\n\tgot:\t%v\n\twant:\t%v", got, test.want)
352 }
353 }
354 }
355
356
357
358 func TestSignedURL_EmulatorHost(t *testing.T) {
359 expires, _ := time.Parse(time.RFC3339, "2002-10-02T10:00:00-05:00")
360 bucketName := "bucket-name"
361 objectName := "obj-name"
362
363 emulatorHost := os.Getenv("STORAGE_EMULATOR_HOST")
364 defer os.Setenv("STORAGE_EMULATOR_HOST", emulatorHost)
365
366 tests := []struct {
367 desc string
368 emulatorHost string
369 now time.Time
370 opts *SignedURLOptions
371
372
373
374
375
376 want string
377 }{
378 {
379 desc: "SignURLV4 creates link to resources in emulator",
380 emulatorHost: "localhost:9000",
381 now: expires.Add(-24 * time.Hour),
382 opts: &SignedURLOptions{
383 GoogleAccessID: "xxx@clientid",
384 PrivateKey: dummyKey("rsa"),
385 Method: "POST",
386 Expires: expires,
387 Scheme: SigningSchemeV4,
388 Insecure: true,
389 },
390 want: "http://localhost:9000/" + bucketName + "/" + objectName +
391 "?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
392 "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
393 "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
394 "&X-Goog-Signature=249c53142e57adf594b4f523a8a1f9c15f29b071e9abc0cf6665dbc5f692fc96fac4ab98bbea4c2397384367bc970a2e1771f2c86624475f3273970ecde8ff6df39d647e5c3f3263bf67a743e211c1958a96775edf53ece1f69ed337f0ab7fdc081c6c2b84e57b0922280d27f1da1bff47e77e3822fb1756e4c5cece9d220e6d0824ab9528e97e54f0cb09b352193b0e895344d894de11b3f5f9a2ec7d8fd6d0a4c487afd1896385a3ab9e8c3fcb3862ec0cad6ec10af1b574078eb7c79b558bcd85449a67079a0ee6da97fcbad074f1bf9fdfbdca12945336a8bd0a3b70b4c7708918cb83d10c7c4ff1f8b73275e9d1ba5d3db91069dffdf81eb7badf4e3c80" +
395 "&X-Goog-SignedHeaders=host",
396 },
397 {
398 desc: "using SigningSchemeV2",
399 emulatorHost: "localhost:9000",
400 now: expires.Add(-24 * time.Hour),
401 opts: &SignedURLOptions{
402 GoogleAccessID: "xxx@clientid",
403 PrivateKey: dummyKey("rsa"),
404 Method: "POST",
405 Expires: expires,
406 Scheme: SigningSchemeV2,
407 },
408 want: "https://localhost:9000/" + bucketName + "/" + objectName +
409 "?Expires=1033570800" +
410 "&GoogleAccessId=xxx%40clientid" +
411 "&Signature=oRi3y2tBTmoDto7FezNx4AjC0RXA6fpJjTBa0hINeVroZ%2ByOeRU8MRwJbKg1IkBbV0IjtlPaGwv5YoUH16UYdipBjCXOS%2B1qgRWyzl8AnzvU%2BfwSXSlCk9zPtHHoBkFT7G4cZQOdDTLRrSG%2FmRJ3K09KEHYg%2Fc6R5Dd92inD1tLE2tiFMyHFs5uQHRMsepY4wrWiIQ4u53tPvk%2Fwiq1%2B9yL6x3QGblhdWwjX0BTVBOxexyKTlwczJW0XlWX8wpcTFfzQnJZuujbhanf2g9MGzSmkv3ylyuQdHMJDYp4Bzq%2FmnkNUg0Vp6iEvh9tyVdRNkwXeg3D8qn%2BFSOxcF%2B9vJw%3D%3D",
412 },
413 {
414 desc: "using VirtualHostedStyle",
415 emulatorHost: "localhost:8000",
416 now: expires.Add(-24 * time.Hour),
417 opts: &SignedURLOptions{
418 GoogleAccessID: "xxx@clientid",
419 PrivateKey: dummyKey("rsa"),
420 Method: "POST",
421 Expires: expires,
422 Scheme: SigningSchemeV4,
423 Style: VirtualHostedStyle(),
424 Insecure: true,
425 },
426 want: "http://" + bucketName + ".localhost:8000/" + objectName +
427 "?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
428 "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
429 "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
430 "&X-Goog-Signature=35e0b9d33901a2518956821175f88c2c4eb3f4461b725af74b37c36d23f8bbe927558ac57b0be40d345f20bca55ba0652d38b7a620f8da68d4f733706ad104da468c3a039459acf35f3022e388760cd49893c998c33fe3ccc8c022d7034ab98bdbdcac4b680bb24ae5ed586a42ee9495a873ffc484e297853a8a3892d0d6385c980cb7e3c5c8bdd4939b4c17105f10fe8b5b9744017bf59431ff176c1550ae1c64ddd6628096eb6895c97c5da4d850aca72c14b7f5018c15b34d4b00ec63ff2ccb688ddbef2d32648e247ffd0137498080f320f293eb811a94fb526227324bbbd01335446388797803e67d802f97b52565deba3d2387ecabf4f3094662236017" +
431 "&X-Goog-SignedHeaders=host",
432 },
433 {
434 desc: "using BucketBoundHostname",
435 emulatorHost: "localhost:8000",
436 now: expires.Add(-24 * time.Hour),
437 opts: &SignedURLOptions{
438 GoogleAccessID: "xxx@clientid",
439 PrivateKey: dummyKey("rsa"),
440 Method: "POST",
441 Expires: expires,
442 Scheme: SigningSchemeV4,
443 Style: BucketBoundHostname("myhost"),
444 },
445 want: "https://" + "myhost/" + objectName +
446 "?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
447 "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
448 "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
449 "&X-Goog-Signature=15fe19f6c61bcbdbd6473c32f2bec29caa8a5fa3b2ce32cfb5329a71edaa0d4e5ffe6469f32ed4c23ca2fbed3882fdf1ed107c6a98c2c4995dda6036c64bae51e6cb542c353618f483832aa1f3ef85342ddadd69c13ad4c69fd3f573ea5cf325a58056e3d5a37005217662af63b49fef8688de3c5c7a2f7b43651a030edd0813eb7f7713989a4c29a8add65133ce652895fea9de7dbc6248ee11b4d7c6c1e152df87700100e896e544ba8eeea96584078f56e699665140b750e90550b9b79633f4e7c8409efa807be5670d6e987eeee04a4180be9b9e30bb8557597beaf390a3805cc602c87a3e34800f8bc01449c3dd10ac2f2263e55e55b91e445052548d5e" +
450 "&X-Goog-SignedHeaders=host",
451 },
452 {
453 desc: "emulator host specifies scheme",
454 emulatorHost: "https://localhost:6000",
455 now: expires.Add(-24 * time.Hour),
456 opts: &SignedURLOptions{
457 GoogleAccessID: "xxx@clientid",
458 PrivateKey: dummyKey("rsa"),
459 Method: "POST",
460 Expires: expires,
461 Scheme: SigningSchemeV4,
462 Insecure: true,
463 },
464 want: "http://localhost:6000/" + bucketName + "/" + objectName +
465 "?X-Goog-Algorithm=GOOG4-RSA-SHA256" +
466 "&X-Goog-Credential=xxx%40clientid%2F20021001%2Fauto%2Fstorage%2Fgoog4_request" +
467 "&X-Goog-Date=20021001T100000Z&X-Goog-Expires=86400" +
468 "&X-Goog-Signature=249c53142e57adf594b4f523a8a1f9c15f29b071e9abc0cf6665dbc5f692fc96fac4ab98bbea4c2397384367bc970a2e1771f2c86624475f3273970ecde8ff6df39d647e5c3f3263bf67a743e211c1958a96775edf53ece1f69ed337f0ab7fdc081c6c2b84e57b0922280d27f1da1bff47e77e3822fb1756e4c5cece9d220e6d0824ab9528e97e54f0cb09b352193b0e895344d894de11b3f5f9a2ec7d8fd6d0a4c487afd1896385a3ab9e8c3fcb3862ec0cad6ec10af1b574078eb7c79b558bcd85449a67079a0ee6da97fcbad074f1bf9fdfbdca12945336a8bd0a3b70b4c7708918cb83d10c7c4ff1f8b73275e9d1ba5d3db91069dffdf81eb7badf4e3c80" +
469 "&X-Goog-SignedHeaders=host",
470 },
471 {
472 desc: "emulator host specifies scheme using SigningSchemeV2",
473 emulatorHost: "https://localhost:8000",
474 now: expires.Add(-24 * time.Hour),
475 opts: &SignedURLOptions{
476 GoogleAccessID: "xxx@clientid",
477 PrivateKey: dummyKey("rsa"),
478 Method: "POST",
479 Expires: expires,
480 Scheme: SigningSchemeV2,
481 },
482 want: "https://localhost:8000/" + bucketName + "/" + objectName +
483 "?Expires=1033570800" +
484 "&GoogleAccessId=xxx%40clientid" +
485 "&Signature=oRi3y2tBTmoDto7FezNx4AjC0RXA6fpJjTBa0hINeVroZ%2ByOeRU8MRwJbKg1IkBbV0IjtlPaGwv5YoUH16UYdipBjCXOS%2B1qgRWyzl8AnzvU%2BfwSXSlCk9zPtHHoBkFT7G4cZQOdDTLRrSG%2FmRJ3K09KEHYg%2Fc6R5Dd92inD1tLE2tiFMyHFs5uQHRMsepY4wrWiIQ4u53tPvk%2Fwiq1%2B9yL6x3QGblhdWwjX0BTVBOxexyKTlwczJW0XlWX8wpcTFfzQnJZuujbhanf2g9MGzSmkv3ylyuQdHMJDYp4Bzq%2FmnkNUg0Vp6iEvh9tyVdRNkwXeg3D8qn%2BFSOxcF%2B9vJw%3D%3D",
486 },
487 }
488 oldUTCNow := utcNow
489 defer func() {
490 utcNow = oldUTCNow
491 }()
492
493 for _, test := range tests {
494 t.Run(test.desc, func(s *testing.T) {
495 utcNow = func() time.Time {
496 return test.now
497 }
498
499 os.Setenv("STORAGE_EMULATOR_HOST", test.emulatorHost)
500
501 got, err := SignedURL(bucketName, objectName, test.opts)
502 if err != nil {
503 s.Fatal(err)
504 }
505
506 if got != test.want {
507 s.Fatalf("\n\tgot:\t%v\n\twant:\t%v", got, test.want)
508 }
509 })
510 }
511 }
512
513 func TestSignedURL_MissingOptions(t *testing.T) {
514 now, _ := time.Parse(time.RFC3339, "2002-10-01T00:00:00-05:00")
515 expires, _ := time.Parse(time.RFC3339, "2002-10-15T00:00:00-05:00")
516 pk := dummyKey("rsa")
517
518 var tests = []struct {
519 opts *SignedURLOptions
520 errMsg string
521 }{
522 {
523 &SignedURLOptions{},
524 "missing required GoogleAccessID",
525 },
526 {
527 &SignedURLOptions{GoogleAccessID: "access_id"},
528 "exactly one of PrivateKey or SignedBytes must be set",
529 },
530 {
531 &SignedURLOptions{
532 GoogleAccessID: "access_id",
533 SignBytes: func(b []byte) ([]byte, error) { return b, nil },
534 PrivateKey: pk,
535 },
536 "exactly one of PrivateKey or SignedBytes must be set",
537 },
538 {
539 &SignedURLOptions{
540 GoogleAccessID: "access_id",
541 PrivateKey: pk,
542 },
543 errMethodNotValid.Error(),
544 },
545 {
546 &SignedURLOptions{
547 GoogleAccessID: "access_id",
548 PrivateKey: pk,
549 Method: "getMethod",
550 },
551 errMethodNotValid.Error(),
552 },
553 {
554 &SignedURLOptions{
555 GoogleAccessID: "access_id",
556 PrivateKey: pk,
557 Method: "get",
558 },
559 "missing required expires",
560 },
561 {
562 &SignedURLOptions{
563 GoogleAccessID: "access_id",
564 SignBytes: func(b []byte) ([]byte, error) { return b, nil },
565 },
566 errMethodNotValid.Error(),
567 },
568 {
569 &SignedURLOptions{
570 GoogleAccessID: "access_id",
571 PrivateKey: pk,
572 Method: "PUT",
573 },
574 "missing required expires",
575 },
576 {
577 &SignedURLOptions{
578 GoogleAccessID: "access_id",
579 PrivateKey: pk,
580 Method: "PUT",
581 Expires: expires,
582 MD5: "invalid",
583 },
584 "invalid MD5 checksum",
585 },
586
587 {
588 &SignedURLOptions{
589 PrivateKey: pk,
590 Method: "GET",
591 Expires: expires,
592 Scheme: SigningSchemeV4,
593 },
594 "missing required GoogleAccessID",
595 },
596 {
597 &SignedURLOptions{
598 GoogleAccessID: "access_id",
599 Method: "GET",
600 Expires: expires,
601 SignBytes: func(b []byte) ([]byte, error) { return b, nil },
602 PrivateKey: pk,
603 Scheme: SigningSchemeV4,
604 },
605 "exactly one of PrivateKey or SignedBytes must be set",
606 },
607 {
608 &SignedURLOptions{
609 GoogleAccessID: "access_id",
610 PrivateKey: pk,
611 Expires: expires,
612 Scheme: SigningSchemeV4,
613 },
614 errMethodNotValid.Error(),
615 },
616 {
617 &SignedURLOptions{
618 GoogleAccessID: "access_id",
619 PrivateKey: pk,
620 Method: "PUT",
621 Scheme: SigningSchemeV4,
622 },
623 "missing required expires",
624 },
625 {
626 &SignedURLOptions{
627 GoogleAccessID: "access_id",
628 PrivateKey: pk,
629 Method: "PUT",
630 Expires: now.Add(time.Hour),
631 MD5: "invalid",
632 Scheme: SigningSchemeV4,
633 },
634 "invalid MD5 checksum",
635 },
636 {
637 &SignedURLOptions{
638 GoogleAccessID: "access_id",
639 PrivateKey: pk,
640 Method: "GET",
641 Expires: expires,
642 Scheme: SigningSchemeV4,
643 },
644 "expires must be within seven days from now",
645 },
646 {
647 &SignedURLOptions{
648 GoogleAccessID: "access_id",
649 PrivateKey: pk,
650 Method: "GET",
651 Expires: now.Add(time.Hour),
652 Scheme: SigningSchemeV2,
653 Style: VirtualHostedStyle(),
654 },
655 "are permitted with SigningSchemeV2",
656 },
657 }
658 oldUTCNow := utcNow
659 defer func() {
660 utcNow = oldUTCNow
661 }()
662 utcNow = func() time.Time {
663 return now
664 }
665
666 for _, test := range tests {
667 _, err := SignedURL("bucket", "name", test.opts)
668 if !strings.Contains(err.Error(), test.errMsg) {
669 t.Errorf("expected err: %v, found: %v", test.errMsg, err)
670 }
671 }
672 }
673
674 func TestPathEncodeV4(t *testing.T) {
675 tests := []struct {
676 input string
677 want string
678 }{
679 {
680 "path/with/slashes",
681 "path/with/slashes",
682 },
683 {
684 "path/with/speci@lchar$&",
685 "path/with/speci%40lchar%24%26",
686 },
687 {
688 "path/with/un_ersc_re/~tilde/sp ace/",
689 "path/with/un_ersc_re/~tilde/sp%20%20ace/",
690 },
691 }
692 for _, test := range tests {
693 if got := pathEncodeV4(test.input); got != test.want {
694 t.Errorf("pathEncodeV4(%q) = %q, want %q", test.input, got, test.want)
695 }
696 }
697 }
698
699 func dummyKey(kind string) []byte {
700 slurp, err := ioutil.ReadFile(fmt.Sprintf("./internal/test/dummy_%s", kind))
701 if err != nil {
702 log.Fatal(err)
703 }
704 return slurp
705 }
706
707 func TestObjectNames(t *testing.T) {
708 t.Parallel()
709
710 const maxLegalLength = 1024
711
712 type testT struct {
713 name, want string
714 }
715 tests := []testT{
716
717 {"foo % bar", "foo%20%25%20bar"},
718 {"foo ? bar", "foo%20%3F%20bar"},
719 {"foo / bar", "foo%20/%20bar"},
720 {"foo %?/ bar", "foo%20%25%3F/%20bar"},
721
722
723 {"타코", "%ED%83%80%EC%BD%94"},
724 {"世界", "%E4%B8%96%E7%95%8C"},
725
726
727 {strings.Repeat("a", maxLegalLength), strings.Repeat("a", maxLegalLength)},
728
729
730 {"foo \u000b bar", "foo%20%0B%20bar"},
731 {"foo \u000c bar", "foo%20%0C%20bar"},
732 {"foo \u0085 bar", "foo%20%C2%85%20bar"},
733 {"foo \u2028 bar", "foo%20%E2%80%A8%20bar"},
734 {"foo \u2029 bar", "foo%20%E2%80%A9%20bar"},
735
736
737 {"foo \u0000 bar", "foo%20%00%20bar"},
738
739
740 {"foo # bar", "foo%20%23%20bar"},
741 {"foo []*? bar", "foo%20%5B%5D%2A%3F%20bar"},
742
743
744 {"foo \u212b bar", "foo%20%E2%84%AB%20bar"},
745 {"foo \u0041\u030a bar", "foo%20A%CC%8A%20bar"},
746 {"foo \u00c5 bar", "foo%20%C3%85%20bar"},
747
748
749 {"foo \u3131\u314f bar", "foo%20%E3%84%B1%E3%85%8F%20bar"},
750 {"foo \u1100\u1161 bar", "foo%20%E1%84%80%E1%85%A1%20bar"},
751 {"foo \uac00 bar", "foo%20%EA%B0%80%20bar"},
752 }
753
754
755 var runes []rune
756 for r := rune(0x01); r <= rune(0x1f); r++ {
757 if r != '\u000a' && r != '\u000d' {
758 runes = append(runes, r)
759 }
760 }
761 tests = append(tests, testT{fmt.Sprintf("foo %s bar", string(runes)), "foo%20%01%02%03%04%05%06%07%08%09%0B%0C%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20bar"})
762
763
764 runes = nil
765 for r := rune(0x7f); r <= rune(0x9f); r++ {
766 runes = append(runes, r)
767 }
768 tests = append(tests, testT{fmt.Sprintf("foo %s bar", string(runes)), "foo%20%7F%C2%80%C2%81%C2%82%C2%83%C2%84%C2%85%C2%86%C2%87%C2%88%C2%89%C2%8A%C2%8B%C2%8C%C2%8D%C2%8E%C2%8F%C2%90%C2%91%C2%92%C2%93%C2%94%C2%95%C2%96%C2%97%C2%98%C2%99%C2%9A%C2%9B%C2%9C%C2%9D%C2%9E%C2%9F%20bar"})
769
770 opts := &SignedURLOptions{
771 GoogleAccessID: "xxx@clientid",
772 PrivateKey: dummyKey("rsa"),
773 Method: "GET",
774 MD5: "ICy5YqxZB1uWSwcVLSNLcA==",
775 Expires: time.Date(2002, time.October, 2, 10, 0, 0, 0, time.UTC),
776 ContentType: "application/json",
777 Headers: []string{"x-goog-header1", "x-goog-header2"},
778 }
779
780 for _, test := range tests {
781 g, err := SignedURL("bucket-name", test.name, opts)
782 if err != nil {
783 t.Errorf("SignedURL(%q) err=%v, want nil", test.name, err)
784 }
785 if w := "/bucket-name/" + test.want; !strings.Contains(g, w) {
786 t.Errorf("SignedURL(%q)=%q, want substring %q", test.name, g, w)
787 }
788 }
789 }
790
791 func TestCondition(t *testing.T) {
792 t.Parallel()
793 gotReq := make(chan *http.Request, 1)
794 hc, close := newTestServer(func(w http.ResponseWriter, r *http.Request) {
795 io.Copy(ioutil.Discard, r.Body)
796 gotReq <- r
797 w.WriteHeader(200)
798 })
799 defer close()
800 ctx := context.Background()
801 c, err := NewClient(ctx, option.WithHTTPClient(hc))
802 if err != nil {
803 t.Fatal(err)
804 }
805
806 obj := c.Bucket("buck").Object("obj")
807 dst := c.Bucket("dstbuck").Object("dst")
808 tests := []struct {
809 fn func() error
810 want string
811 }{
812 {
813 func() error {
814 _, err := obj.Generation(1234).NewReader(ctx)
815 return err
816 },
817 "GET /buck/obj?generation=1234",
818 },
819 {
820 func() error {
821 _, err := obj.If(Conditions{MetagenerationNotMatch: 1234}).Attrs(ctx)
822 return err
823 },
824 "GET /storage/v1/b/buck/o/obj?alt=json&ifMetagenerationNotMatch=1234&prettyPrint=false&projection=full",
825 },
826 {
827 func() error {
828 _, err := obj.If(Conditions{MetagenerationMatch: 1234}).Update(ctx, ObjectAttrsToUpdate{})
829 return err
830 },
831 "PATCH /storage/v1/b/buck/o/obj?alt=json&ifMetagenerationMatch=1234&prettyPrint=false&projection=full",
832 },
833 {
834 func() error { return obj.Generation(1234).Delete(ctx) },
835 "DELETE /storage/v1/b/buck/o/obj?alt=json&generation=1234&prettyPrint=false",
836 },
837 {
838 func() error {
839 w := obj.If(Conditions{GenerationMatch: 1234}).NewWriter(ctx)
840 w.ContentType = "text/plain"
841 return w.Close()
842 },
843 "POST /upload/storage/v1/b/buck/o?alt=json&ifGenerationMatch=1234&name=obj&prettyPrint=false&projection=full&uploadType=multipart",
844 },
845 {
846 func() error {
847 w := obj.If(Conditions{DoesNotExist: true}).NewWriter(ctx)
848 w.ContentType = "text/plain"
849 return w.Close()
850 },
851 "POST /upload/storage/v1/b/buck/o?alt=json&ifGenerationMatch=0&name=obj&prettyPrint=false&projection=full&uploadType=multipart",
852 },
853 {
854 func() error {
855 _, err := dst.If(Conditions{MetagenerationMatch: 5678}).CopierFrom(obj.If(Conditions{GenerationMatch: 1234})).Run(ctx)
856 return err
857 },
858 "POST /storage/v1/b/buck/o/obj/rewriteTo/b/dstbuck/o/dst?alt=json&ifMetagenerationMatch=5678&ifSourceGenerationMatch=1234&prettyPrint=false&projection=full",
859 },
860 }
861
862 for i, tt := range tests {
863 if err := tt.fn(); err != nil && err != io.EOF {
864 t.Error(err)
865 continue
866 }
867 select {
868 case r := <-gotReq:
869 got := r.Method + " " + r.RequestURI
870 if got != tt.want {
871 t.Errorf("%d. RequestURI = %q; want %q", i, got, tt.want)
872 }
873 case <-time.After(5 * time.Second):
874 t.Fatalf("%d. timeout", i)
875 }
876 if err != nil {
877 t.Fatal(err)
878 }
879 }
880
881 readerTests := []struct {
882 fn func() error
883 want string
884 }{
885 {
886 func() error {
887 _, err := obj.If(Conditions{GenerationMatch: 1234}).NewReader(ctx)
888 return err
889 },
890 "x-goog-if-generation-match: 1234, x-goog-if-metageneration-match: ",
891 },
892 {
893 func() error {
894 _, err := obj.If(Conditions{MetagenerationMatch: 5}).NewReader(ctx)
895 return err
896 },
897 "x-goog-if-generation-match: , x-goog-if-metageneration-match: 5",
898 },
899 {
900 func() error {
901 _, err := obj.If(Conditions{GenerationMatch: 1234, MetagenerationMatch: 5}).NewReader(ctx)
902 return err
903 },
904 "x-goog-if-generation-match: 1234, x-goog-if-metageneration-match: 5",
905 },
906 }
907
908 for i, tt := range readerTests {
909 if err := tt.fn(); err != nil && err != io.EOF {
910 t.Error(err)
911 continue
912 }
913
914 select {
915 case r := <-gotReq:
916 generationConds := r.Header.Get("x-goog-if-generation-match")
917 metagenerationConds := r.Header.Get("x-goog-if-metageneration-match")
918 got := fmt.Sprintf(
919 "x-goog-if-generation-match: %s, x-goog-if-metageneration-match: %s",
920 generationConds,
921 metagenerationConds,
922 )
923 if got != tt.want {
924 t.Errorf("%d. RequestHeaders = %q; want %q", i, got, tt.want)
925 }
926 case <-time.After(5 * time.Second):
927 t.Fatalf("%d. timeout", i)
928 }
929 if err != nil {
930 t.Fatal(err)
931 }
932 }
933
934
935 err = obj.Generation(1234).NewWriter(ctx).Close()
936 if err == nil || !strings.Contains(err.Error(), "storage: generation not supported") {
937 t.Errorf("want error about unsupported generation; got %v", err)
938 }
939 }
940
941 func TestConditionErrors(t *testing.T) {
942 t.Parallel()
943 for _, conds := range []Conditions{
944 {GenerationMatch: 0},
945 {DoesNotExist: false},
946 {GenerationMatch: 1, GenerationNotMatch: 2},
947 {GenerationNotMatch: 2, DoesNotExist: true},
948 {MetagenerationMatch: 1, MetagenerationNotMatch: 2},
949 } {
950 if err := conds.validate(""); err == nil {
951 t.Errorf("%+v: got nil, want error", conds)
952 }
953 }
954 }
955
956 func expectedAttempts(value int) *int {
957 return &value
958 }
959
960
961
962 func TestObjectRetryer(t *testing.T) {
963 testCases := []struct {
964 name string
965 call func(o *ObjectHandle) *ObjectHandle
966 want *retryConfig
967 }{
968 {
969 name: "all defaults",
970 call: func(o *ObjectHandle) *ObjectHandle {
971 return o.Retryer()
972 },
973 want: &retryConfig{},
974 },
975 {
976 name: "set all options",
977 call: func(o *ObjectHandle) *ObjectHandle {
978 return o.Retryer(
979 WithBackoff(gax.Backoff{
980 Initial: 2 * time.Second,
981 Max: 30 * time.Second,
982 Multiplier: 3,
983 }),
984 WithMaxAttempts(5),
985 WithPolicy(RetryAlways),
986 WithErrorFunc(func(err error) bool { return false }))
987 },
988 want: &retryConfig{
989 backoff: &gax.Backoff{
990 Initial: 2 * time.Second,
991 Max: 30 * time.Second,
992 Multiplier: 3,
993 },
994 maxAttempts: expectedAttempts(5),
995 policy: RetryAlways,
996 shouldRetry: func(err error) bool { return false },
997 },
998 },
999 {
1000 name: "set some backoff options",
1001 call: func(o *ObjectHandle) *ObjectHandle {
1002 return o.Retryer(
1003 WithBackoff(gax.Backoff{
1004 Multiplier: 3,
1005 }))
1006 },
1007 want: &retryConfig{
1008 backoff: &gax.Backoff{
1009 Multiplier: 3,
1010 }},
1011 },
1012 {
1013 name: "set policy only",
1014 call: func(o *ObjectHandle) *ObjectHandle {
1015 return o.Retryer(WithPolicy(RetryNever))
1016 },
1017 want: &retryConfig{
1018 policy: RetryNever,
1019 },
1020 },
1021 {
1022 name: "set max retry attempts only",
1023 call: func(o *ObjectHandle) *ObjectHandle {
1024 return o.Retryer(WithMaxAttempts(11))
1025 },
1026 want: &retryConfig{
1027 maxAttempts: expectedAttempts(11),
1028 },
1029 },
1030 {
1031 name: "set ErrorFunc only",
1032 call: func(o *ObjectHandle) *ObjectHandle {
1033 return o.Retryer(
1034 WithErrorFunc(func(err error) bool { return false }))
1035 },
1036 want: &retryConfig{
1037 shouldRetry: func(err error) bool { return false },
1038 },
1039 },
1040 }
1041 for _, tc := range testCases {
1042 t.Run(tc.name, func(s *testing.T) {
1043 o := tc.call(&ObjectHandle{})
1044 if diff := cmp.Diff(
1045 o.retry,
1046 tc.want,
1047 cmp.AllowUnexported(retryConfig{}, gax.Backoff{}),
1048
1049
1050 cmp.Comparer(func(a, b func(err error) bool) bool {
1051 return (a == nil && b == nil) || (a != nil && b != nil)
1052 }),
1053 ); diff != "" {
1054 s.Fatalf("retry not configured correctly: %v", diff)
1055 }
1056 })
1057 }
1058 }
1059
1060
1061
1062 func TestClientSetRetry(t *testing.T) {
1063 testCases := []struct {
1064 name string
1065 clientOptions []RetryOption
1066 want *retryConfig
1067 }{
1068 {
1069 name: "all defaults",
1070 clientOptions: []RetryOption{},
1071 want: &retryConfig{},
1072 },
1073 {
1074 name: "set all options",
1075 clientOptions: []RetryOption{
1076 WithBackoff(gax.Backoff{
1077 Initial: 2 * time.Second,
1078 Max: 30 * time.Second,
1079 Multiplier: 3,
1080 }),
1081 WithMaxAttempts(5),
1082 WithPolicy(RetryAlways),
1083 WithErrorFunc(func(err error) bool { return false }),
1084 },
1085 want: &retryConfig{
1086 backoff: &gax.Backoff{
1087 Initial: 2 * time.Second,
1088 Max: 30 * time.Second,
1089 Multiplier: 3,
1090 },
1091 maxAttempts: expectedAttempts(5),
1092 policy: RetryAlways,
1093 shouldRetry: func(err error) bool { return false },
1094 },
1095 },
1096 {
1097 name: "set some backoff options",
1098 clientOptions: []RetryOption{
1099 WithBackoff(gax.Backoff{
1100 Multiplier: 3,
1101 }),
1102 },
1103 want: &retryConfig{
1104 backoff: &gax.Backoff{
1105 Multiplier: 3,
1106 }},
1107 },
1108 {
1109 name: "set policy only",
1110 clientOptions: []RetryOption{
1111 WithPolicy(RetryNever),
1112 },
1113 want: &retryConfig{
1114 policy: RetryNever,
1115 },
1116 },
1117 {
1118 name: "set max retry attempts only",
1119 clientOptions: []RetryOption{
1120 WithMaxAttempts(7),
1121 },
1122 want: &retryConfig{
1123 maxAttempts: expectedAttempts(7),
1124 },
1125 },
1126 {
1127 name: "set ErrorFunc only",
1128 clientOptions: []RetryOption{
1129 WithErrorFunc(func(err error) bool { return false }),
1130 },
1131 want: &retryConfig{
1132 shouldRetry: func(err error) bool { return false },
1133 },
1134 },
1135 }
1136 for _, tc := range testCases {
1137 t.Run(tc.name, func(s *testing.T) {
1138 c, err := NewClient(context.Background(), option.WithoutAuthentication())
1139 if err != nil {
1140 t.Fatalf("NewClient: %v", err)
1141 }
1142 defer c.Close()
1143 c.SetRetry(tc.clientOptions...)
1144
1145 if diff := cmp.Diff(
1146 c.retry,
1147 tc.want,
1148 cmp.AllowUnexported(retryConfig{}, gax.Backoff{}),
1149
1150
1151 cmp.Comparer(func(a, b func(err error) bool) bool {
1152 return (a == nil && b == nil) || (a != nil && b != nil)
1153 }),
1154 ); diff != "" {
1155 s.Fatalf("retry not configured correctly: %v", diff)
1156 }
1157 })
1158 }
1159 }
1160
1161
1162
1163 func TestRetryer(t *testing.T) {
1164 testCases := []struct {
1165 name string
1166 clientOptions []RetryOption
1167 bucketOptions []RetryOption
1168 objectOptions []RetryOption
1169 want *retryConfig
1170 }{
1171 {
1172 name: "no retries",
1173 want: nil,
1174 },
1175 {
1176 name: "object retryer configures retry",
1177 objectOptions: []RetryOption{
1178 WithPolicy(RetryAlways),
1179 WithMaxAttempts(5),
1180 WithErrorFunc(ShouldRetry),
1181 },
1182 want: &retryConfig{
1183 shouldRetry: ShouldRetry,
1184 maxAttempts: expectedAttempts(5),
1185 policy: RetryAlways,
1186 },
1187 },
1188 {
1189 name: "bucket retryer configures retry",
1190 bucketOptions: []RetryOption{
1191 WithBackoff(gax.Backoff{
1192 Initial: time.Minute,
1193 Max: time.Hour,
1194 Multiplier: 6,
1195 }),
1196 WithPolicy(RetryAlways),
1197 WithMaxAttempts(11),
1198 WithErrorFunc(ShouldRetry),
1199 },
1200 want: &retryConfig{
1201 backoff: &gax.Backoff{
1202 Initial: time.Minute,
1203 Max: time.Hour,
1204 Multiplier: 6,
1205 },
1206 shouldRetry: ShouldRetry,
1207 maxAttempts: expectedAttempts(11),
1208 policy: RetryAlways,
1209 },
1210 },
1211 {
1212 name: "client retryer configures retry",
1213 clientOptions: []RetryOption{
1214 WithBackoff(gax.Backoff{
1215 Initial: time.Minute,
1216 Max: time.Hour,
1217 Multiplier: 6,
1218 }),
1219 WithPolicy(RetryAlways),
1220 WithMaxAttempts(7),
1221 WithErrorFunc(ShouldRetry),
1222 },
1223 want: &retryConfig{
1224 backoff: &gax.Backoff{
1225 Initial: time.Minute,
1226 Max: time.Hour,
1227 Multiplier: 6,
1228 },
1229 shouldRetry: ShouldRetry,
1230 maxAttempts: expectedAttempts(7),
1231 policy: RetryAlways,
1232 },
1233 },
1234 {
1235 name: "object retryer overrides bucket retryer",
1236 bucketOptions: []RetryOption{
1237 WithPolicy(RetryAlways),
1238 },
1239 objectOptions: []RetryOption{
1240 WithPolicy(RetryNever),
1241 WithMaxAttempts(5),
1242 WithErrorFunc(ShouldRetry),
1243 },
1244 want: &retryConfig{
1245 policy: RetryNever,
1246 maxAttempts: expectedAttempts(5),
1247 shouldRetry: ShouldRetry,
1248 },
1249 },
1250 {
1251 name: "object retryer overrides client retryer",
1252 clientOptions: []RetryOption{
1253 WithPolicy(RetryAlways),
1254 },
1255 objectOptions: []RetryOption{
1256 WithPolicy(RetryNever),
1257 WithMaxAttempts(11),
1258 WithErrorFunc(ShouldRetry),
1259 },
1260 want: &retryConfig{
1261 policy: RetryNever,
1262 maxAttempts: expectedAttempts(11),
1263 shouldRetry: ShouldRetry,
1264 },
1265 },
1266 {
1267 name: "bucket retryer overrides client retryer",
1268 clientOptions: []RetryOption{
1269 WithBackoff(gax.Backoff{
1270 Initial: time.Minute,
1271 Max: time.Hour,
1272 Multiplier: 6,
1273 }),
1274 WithPolicy(RetryAlways),
1275 },
1276 bucketOptions: []RetryOption{
1277 WithBackoff(gax.Backoff{
1278 Initial: time.Nanosecond,
1279 Max: time.Microsecond,
1280 }),
1281 WithErrorFunc(ShouldRetry),
1282 WithMaxAttempts(5),
1283 },
1284 want: &retryConfig{
1285 policy: RetryAlways,
1286 maxAttempts: expectedAttempts(5),
1287 shouldRetry: ShouldRetry,
1288 backoff: &gax.Backoff{
1289 Initial: time.Nanosecond,
1290 Max: time.Microsecond,
1291 },
1292 },
1293 },
1294 {
1295 name: "object retryer overrides bucket retryer backoff options",
1296 bucketOptions: []RetryOption{
1297 WithBackoff(gax.Backoff{
1298 Initial: time.Minute,
1299 Max: time.Hour,
1300 Multiplier: 6,
1301 }),
1302 },
1303 objectOptions: []RetryOption{
1304 WithBackoff(gax.Backoff{
1305 Initial: time.Nanosecond,
1306 Max: time.Microsecond,
1307 }),
1308 },
1309 want: &retryConfig{
1310 backoff: &gax.Backoff{
1311 Initial: time.Nanosecond,
1312 Max: time.Microsecond,
1313 },
1314 },
1315 },
1316 {
1317 name: "object retryer does not override bucket retryer if option is not set",
1318 bucketOptions: []RetryOption{
1319 WithPolicy(RetryNever),
1320 WithErrorFunc(ShouldRetry),
1321 WithMaxAttempts(5),
1322 },
1323 objectOptions: []RetryOption{
1324 WithBackoff(gax.Backoff{
1325 Initial: time.Nanosecond,
1326 Max: time.Second,
1327 }),
1328 },
1329 want: &retryConfig{
1330 policy: RetryNever,
1331 maxAttempts: expectedAttempts(5),
1332 shouldRetry: ShouldRetry,
1333 backoff: &gax.Backoff{
1334 Initial: time.Nanosecond,
1335 Max: time.Second,
1336 },
1337 },
1338 },
1339 {
1340 name: "object's backoff completely overwrites bucket's backoff",
1341 bucketOptions: []RetryOption{
1342 WithBackoff(gax.Backoff{
1343 Initial: time.Hour,
1344 }),
1345 },
1346 objectOptions: []RetryOption{
1347 WithBackoff(gax.Backoff{
1348 Multiplier: 4,
1349 }),
1350 },
1351 want: &retryConfig{
1352 backoff: &gax.Backoff{
1353 Multiplier: 4,
1354 },
1355 },
1356 },
1357 }
1358 for _, tc := range testCases {
1359 t.Run(tc.name, func(s *testing.T) {
1360 ctx := context.Background()
1361 c, err := NewClient(ctx, option.WithoutAuthentication())
1362 if err != nil {
1363 t.Fatalf("NewClient: %v", err)
1364 }
1365 defer c.Close()
1366 if len(tc.clientOptions) > 0 {
1367 c.SetRetry(tc.clientOptions...)
1368 }
1369 b := c.Bucket("buck")
1370 if len(tc.bucketOptions) > 0 {
1371 b = b.Retryer(tc.bucketOptions...)
1372 }
1373 o := b.Object("obj")
1374 if len(tc.objectOptions) > 0 {
1375 o = o.Retryer(tc.objectOptions...)
1376 }
1377
1378 configHandleCases := []struct {
1379 r *retryConfig
1380 name string
1381 want *retryConfig
1382 }{
1383 {
1384 name: "object.retry",
1385 r: o.retry,
1386 want: tc.want,
1387 },
1388 {
1389 name: "object.ACL()",
1390 r: o.ACL().retry,
1391 want: tc.want,
1392 },
1393 {
1394 name: "bucket.ACL()",
1395 r: b.ACL().retry,
1396 want: b.retry,
1397 },
1398 {
1399 name: "bucket.DefaultObjectACL()",
1400 r: b.DefaultObjectACL().retry,
1401 want: b.retry,
1402 },
1403 {
1404 name: "client.HMACKeyHandle()",
1405 r: c.HMACKeyHandle("pID", "accessID").retry,
1406 want: c.retry,
1407 },
1408 }
1409 for _, ac := range configHandleCases {
1410 s.Run(ac.name, func(ss *testing.T) {
1411 if diff := cmp.Diff(
1412 ac.want,
1413 ac.r,
1414 cmp.AllowUnexported(retryConfig{}, gax.Backoff{}),
1415
1416
1417 cmp.Comparer(func(a, b func(err error) bool) bool {
1418 return (a == nil && b == nil) || (a != nil && b != nil)
1419 }),
1420 ); diff != "" {
1421 ss.Fatalf("retry not configured correctly: %v", diff)
1422 }
1423 })
1424 }
1425 })
1426 }
1427 }
1428
1429
1430 func TestObjectCompose(t *testing.T) {
1431 t.Parallel()
1432 gotURL := make(chan string, 1)
1433 gotBody := make(chan []byte, 1)
1434 hc, close := newTestServer(func(w http.ResponseWriter, r *http.Request) {
1435 body, _ := ioutil.ReadAll(r.Body)
1436 gotURL <- r.URL.String()
1437 gotBody <- body
1438 w.Write([]byte("{}"))
1439 })
1440 defer close()
1441 ctx := context.Background()
1442 c, err := NewClient(ctx, option.WithHTTPClient(hc))
1443 if err != nil {
1444 t.Fatal(err)
1445 }
1446
1447 testCases := []struct {
1448 desc string
1449 dst *ObjectHandle
1450 srcs []*ObjectHandle
1451 attrs *ObjectAttrs
1452 sendCRC32C bool
1453 wantReq raw.ComposeRequest
1454 wantURL string
1455 wantErr bool
1456 }{
1457 {
1458 desc: "basic case",
1459 dst: c.Bucket("foo").Object("bar"),
1460 srcs: []*ObjectHandle{
1461 c.Bucket("foo").Object("baz"),
1462 c.Bucket("foo").Object("quux"),
1463 },
1464 wantURL: "/storage/v1/b/foo/o/bar/compose?alt=json&prettyPrint=false",
1465 wantReq: raw.ComposeRequest{
1466 Destination: &raw.Object{Bucket: "foo"},
1467 SourceObjects: []*raw.ComposeRequestSourceObjects{
1468 {Name: "baz"},
1469 {Name: "quux"},
1470 },
1471 },
1472 },
1473 {
1474 desc: "with object attrs",
1475 dst: c.Bucket("foo").Object("bar"),
1476 srcs: []*ObjectHandle{
1477 c.Bucket("foo").Object("baz"),
1478 c.Bucket("foo").Object("quux"),
1479 },
1480 attrs: &ObjectAttrs{
1481 Name: "not-bar",
1482 ContentType: "application/json",
1483 },
1484 wantURL: "/storage/v1/b/foo/o/bar/compose?alt=json&prettyPrint=false",
1485 wantReq: raw.ComposeRequest{
1486 Destination: &raw.Object{
1487 Bucket: "foo",
1488 Name: "not-bar",
1489 ContentType: "application/json",
1490 },
1491 SourceObjects: []*raw.ComposeRequestSourceObjects{
1492 {Name: "baz"},
1493 {Name: "quux"},
1494 },
1495 },
1496 },
1497 {
1498 desc: "with conditions",
1499 dst: c.Bucket("foo").Object("bar").If(Conditions{
1500 GenerationMatch: 12,
1501 MetagenerationMatch: 34,
1502 }),
1503 srcs: []*ObjectHandle{
1504 c.Bucket("foo").Object("baz").Generation(56),
1505 c.Bucket("foo").Object("quux").If(Conditions{GenerationMatch: 78}),
1506 },
1507 wantURL: "/storage/v1/b/foo/o/bar/compose?alt=json&ifGenerationMatch=12&ifMetagenerationMatch=34&prettyPrint=false",
1508 wantReq: raw.ComposeRequest{
1509 Destination: &raw.Object{Bucket: "foo"},
1510 SourceObjects: []*raw.ComposeRequestSourceObjects{
1511 {
1512 Name: "baz",
1513 Generation: 56,
1514 },
1515 {
1516 Name: "quux",
1517 ObjectPreconditions: &raw.ComposeRequestSourceObjectsObjectPreconditions{
1518 IfGenerationMatch: 78,
1519 },
1520 },
1521 },
1522 },
1523 },
1524 {
1525 desc: "with crc32c",
1526 dst: c.Bucket("foo").Object("bar"),
1527 srcs: []*ObjectHandle{
1528 c.Bucket("foo").Object("baz"),
1529 c.Bucket("foo").Object("quux"),
1530 },
1531 attrs: &ObjectAttrs{
1532 CRC32C: 42,
1533 },
1534 sendCRC32C: true,
1535 wantURL: "/storage/v1/b/foo/o/bar/compose?alt=json&prettyPrint=false",
1536 wantReq: raw.ComposeRequest{
1537 Destination: &raw.Object{Bucket: "foo", Crc32c: "AAAAKg=="},
1538 SourceObjects: []*raw.ComposeRequestSourceObjects{
1539 {Name: "baz"},
1540 {Name: "quux"},
1541 },
1542 },
1543 },
1544 {
1545 desc: "no sources",
1546 dst: c.Bucket("foo").Object("bar"),
1547 wantErr: true,
1548 },
1549 {
1550 desc: "destination, no bucket",
1551 dst: c.Bucket("").Object("bar"),
1552 srcs: []*ObjectHandle{
1553 c.Bucket("foo").Object("baz"),
1554 },
1555 wantErr: true,
1556 },
1557 {
1558 desc: "destination, no object",
1559 dst: c.Bucket("foo").Object(""),
1560 srcs: []*ObjectHandle{
1561 c.Bucket("foo").Object("baz"),
1562 },
1563 wantErr: true,
1564 },
1565 {
1566 desc: "source, different bucket",
1567 dst: c.Bucket("foo").Object("bar"),
1568 srcs: []*ObjectHandle{
1569 c.Bucket("otherbucket").Object("baz"),
1570 },
1571 wantErr: true,
1572 },
1573 {
1574 desc: "source, no object",
1575 dst: c.Bucket("foo").Object("bar"),
1576 srcs: []*ObjectHandle{
1577 c.Bucket("foo").Object(""),
1578 },
1579 wantErr: true,
1580 },
1581 {
1582 desc: "destination, bad condition",
1583 dst: c.Bucket("foo").Object("bar").Generation(12),
1584 srcs: []*ObjectHandle{
1585 c.Bucket("foo").Object("baz"),
1586 },
1587 wantErr: true,
1588 },
1589 {
1590 desc: "source, bad condition",
1591 dst: c.Bucket("foo").Object("bar"),
1592 srcs: []*ObjectHandle{
1593 c.Bucket("foo").Object("baz").If(Conditions{MetagenerationMatch: 12}),
1594 },
1595 wantErr: true,
1596 },
1597 }
1598
1599 for _, tt := range testCases {
1600 composer := tt.dst.ComposerFrom(tt.srcs...)
1601 if tt.attrs != nil {
1602 composer.ObjectAttrs = *tt.attrs
1603 }
1604 composer.SendCRC32C = tt.sendCRC32C
1605 _, err := composer.Run(ctx)
1606 if gotErr := err != nil; gotErr != tt.wantErr {
1607 t.Errorf("%s: got error %v; want err %t", tt.desc, err, tt.wantErr)
1608 continue
1609 }
1610 if tt.wantErr {
1611 continue
1612 }
1613 u, body := <-gotURL, <-gotBody
1614 if u != tt.wantURL {
1615 t.Errorf("%s: request URL\ngot %q\nwant %q", tt.desc, u, tt.wantURL)
1616 }
1617 var req raw.ComposeRequest
1618 if err := json.Unmarshal(body, &req); err != nil {
1619 t.Errorf("%s: json.Unmarshal %v (body %s)", tt.desc, err, body)
1620 }
1621 if !testutil.Equal(req, tt.wantReq) {
1622
1623 wantReq, _ := json.Marshal(tt.wantReq)
1624 t.Errorf("%s: request body\ngot %s\nwant %s", tt.desc, body, wantReq)
1625 }
1626 }
1627 }
1628
1629
1630
1631 func TestEmptyObjectIterator(t *testing.T) {
1632 t.Parallel()
1633 hClient, close := newTestServer(func(w http.ResponseWriter, r *http.Request) {
1634 io.Copy(ioutil.Discard, r.Body)
1635 fmt.Fprintf(w, "{}")
1636 })
1637 defer close()
1638 ctx := context.Background()
1639 client, err := NewClient(ctx, option.WithHTTPClient(hClient))
1640 if err != nil {
1641 t.Fatal(err)
1642 }
1643 it := client.Bucket("b").Objects(ctx, nil)
1644 _, err = it.Next()
1645 if err != iterator.Done {
1646 t.Errorf("got %v, want Done", err)
1647 }
1648 }
1649
1650
1651
1652 func TestEmptyBucketIterator(t *testing.T) {
1653 t.Parallel()
1654 hClient, close := newTestServer(func(w http.ResponseWriter, r *http.Request) {
1655 io.Copy(ioutil.Discard, r.Body)
1656 fmt.Fprintf(w, "{}")
1657 })
1658 defer close()
1659 ctx := context.Background()
1660 client, err := NewClient(ctx, option.WithHTTPClient(hClient))
1661 if err != nil {
1662 t.Fatal(err)
1663 }
1664 it := client.Buckets(ctx, "project")
1665 _, err = it.Next()
1666 if err != iterator.Done {
1667 t.Errorf("got %v, want Done", err)
1668 }
1669
1670 }
1671
1672 func TestCodecUint32(t *testing.T) {
1673 t.Parallel()
1674 for _, u := range []uint32{0, 1, 256, 0xFFFFFFFF} {
1675 s := encodeUint32(u)
1676 d, err := decodeUint32(s)
1677 if err != nil {
1678 t.Fatal(err)
1679 }
1680 if d != u {
1681 t.Errorf("got %d, want input %d", d, u)
1682 }
1683 }
1684 }
1685
1686 func TestUserProject(t *testing.T) {
1687
1688 t.Parallel()
1689 ctx := context.Background()
1690 gotURL := make(chan *url.URL, 1)
1691 hClient, close := newTestServer(func(w http.ResponseWriter, r *http.Request) {
1692 io.Copy(ioutil.Discard, r.Body)
1693 gotURL <- r.URL
1694 if strings.Contains(r.URL.String(), "/rewriteTo/") {
1695 res := &raw.RewriteResponse{Done: true}
1696 bytes, err := res.MarshalJSON()
1697 if err != nil {
1698 t.Fatal(err)
1699 }
1700 w.Write(bytes)
1701 } else {
1702 fmt.Fprintf(w, "{}")
1703 }
1704 })
1705 defer close()
1706 client, err := NewClient(ctx, option.WithHTTPClient(hClient))
1707 if err != nil {
1708 t.Fatal(err)
1709 }
1710
1711 re := regexp.MustCompile(`\buserProject=p\b`)
1712 b := client.Bucket("b").UserProject("p")
1713 o := b.Object("o")
1714
1715 check := func(msg string, f func()) {
1716 f()
1717 select {
1718 case u := <-gotURL:
1719 if !re.MatchString(u.RawQuery) {
1720 t.Errorf("%s: query string %q does not contain userProject", msg, u.RawQuery)
1721 }
1722 case <-time.After(2 * time.Second):
1723 t.Errorf("%s: timed out", msg)
1724 }
1725 }
1726
1727 check("buckets.delete", func() { b.Delete(ctx) })
1728 check("buckets.get", func() { b.Attrs(ctx) })
1729 check("buckets.patch", func() { b.Update(ctx, BucketAttrsToUpdate{}) })
1730 check("storage.objects.compose", func() { o.ComposerFrom(b.Object("x")).Run(ctx) })
1731 check("storage.objects.delete", func() { o.Delete(ctx) })
1732 check("storage.objects.get", func() { o.Attrs(ctx) })
1733 check("storage.objects.insert", func() { o.NewWriter(ctx).Close() })
1734 check("storage.objects.list", func() { b.Objects(ctx, nil).Next() })
1735 check("storage.objects.patch", func() { o.Update(ctx, ObjectAttrsToUpdate{}) })
1736 check("storage.objects.rewrite", func() { o.CopierFrom(b.Object("x")).Run(ctx) })
1737 check("storage.objectAccessControls.list", func() { o.ACL().List(ctx) })
1738 check("storage.objectAccessControls.update", func() { o.ACL().Set(ctx, "", "") })
1739 check("storage.objectAccessControls.delete", func() { o.ACL().Delete(ctx, "") })
1740 check("storage.bucketAccessControls.list", func() { b.ACL().List(ctx) })
1741 check("storage.bucketAccessControls.update", func() { b.ACL().Set(ctx, "", "") })
1742 check("storage.bucketAccessControls.delete", func() { b.ACL().Delete(ctx, "") })
1743 check("storage.defaultObjectAccessControls.list",
1744 func() { b.DefaultObjectACL().List(ctx) })
1745 check("storage.defaultObjectAccessControls.update",
1746 func() { b.DefaultObjectACL().Set(ctx, "", "") })
1747 check("storage.defaultObjectAccessControls.delete",
1748 func() { b.DefaultObjectACL().Delete(ctx, "") })
1749 check("buckets.getIamPolicy", func() { b.IAM().Policy(ctx) })
1750 check("buckets.setIamPolicy", func() {
1751 p := &iam.Policy{}
1752 p.Add("m", iam.Owner)
1753 b.IAM().SetPolicy(ctx, p)
1754 })
1755 check("buckets.testIamPermissions", func() { b.IAM().TestPermissions(ctx, nil) })
1756 check("storage.notifications.insert", func() {
1757 b.AddNotification(ctx, &Notification{TopicProjectID: "p", TopicID: "t"})
1758 })
1759 check("storage.notifications.delete", func() { b.DeleteNotification(ctx, "n") })
1760 check("storage.notifications.list", func() { b.Notifications(ctx) })
1761 }
1762
1763 func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*http.Client, func()) {
1764 ts := httptest.NewTLSServer(http.HandlerFunc(handler))
1765 tlsConf := &tls.Config{InsecureSkipVerify: true}
1766 tr := &http.Transport{
1767 TLSClientConfig: tlsConf,
1768 DialTLS: func(netw, addr string) (net.Conn, error) {
1769 return tls.Dial("tcp", ts.Listener.Addr().String(), tlsConf)
1770 },
1771 }
1772 return &http.Client{Transport: tr}, func() {
1773 tr.CloseIdleConnections()
1774 ts.Close()
1775 }
1776 }
1777
1778 func TestRawObjectToObjectAttrs(t *testing.T) {
1779 t.Parallel()
1780 tests := []struct {
1781 in *raw.Object
1782 want *ObjectAttrs
1783 }{
1784 {in: nil, want: nil},
1785 {
1786 in: &raw.Object{
1787 Bucket: "Test",
1788 ContentLanguage: "en-us",
1789 ContentType: "video/mpeg",
1790 CustomTime: "2020-08-25T19:33:36Z",
1791 EventBasedHold: false,
1792 Etag: "Zkyw9ACJZUvcYmlFaKGChzhmtnE/dt1zHSfweiWpwzdGsqXwuJZqiD0",
1793 Generation: 7,
1794 Md5Hash: "MTQ2ODNjYmE0NDRkYmNjNmRiMjk3NjQ1ZTY4M2Y1YzE=",
1795 Name: "foo.mp4",
1796 RetentionExpirationTime: "2019-03-31T19:33:36Z",
1797 Size: 1 << 20,
1798 TimeCreated: "2019-03-31T19:32:10Z",
1799 TimeDeleted: "2019-03-31T19:33:39Z",
1800 TemporaryHold: true,
1801 ComponentCount: 2,
1802 },
1803 want: &ObjectAttrs{
1804 Bucket: "Test",
1805 Created: time.Date(2019, 3, 31, 19, 32, 10, 0, time.UTC),
1806 ContentLanguage: "en-us",
1807 ContentType: "video/mpeg",
1808 CustomTime: time.Date(2020, 8, 25, 19, 33, 36, 0, time.UTC),
1809 Deleted: time.Date(2019, 3, 31, 19, 33, 39, 0, time.UTC),
1810 EventBasedHold: false,
1811 Etag: "Zkyw9ACJZUvcYmlFaKGChzhmtnE/dt1zHSfweiWpwzdGsqXwuJZqiD0",
1812 Generation: 7,
1813 MD5: []byte("14683cba444dbcc6db297645e683f5c1"),
1814 Name: "foo.mp4",
1815 RetentionExpirationTime: time.Date(2019, 3, 31, 19, 33, 36, 0, time.UTC),
1816 Size: 1 << 20,
1817 TemporaryHold: true,
1818 ComponentCount: 2,
1819 },
1820 },
1821 }
1822
1823 for i, tt := range tests {
1824 got := newObject(tt.in)
1825 if diff := testutil.Diff(got, tt.want); diff != "" {
1826 t.Errorf("#%d: newObject mismatches:\ngot=-, want=+:\n%s", i, diff)
1827 }
1828 }
1829 }
1830
1831 func TestObjectAttrsToRawObject(t *testing.T) {
1832 t.Parallel()
1833 bucketName := "the-bucket"
1834 in := &ObjectAttrs{
1835 Bucket: "Test",
1836 Created: time.Date(2019, 3, 31, 19, 32, 10, 0, time.UTC),
1837 ContentLanguage: "en-us",
1838 ContentType: "video/mpeg",
1839 Deleted: time.Date(2019, 3, 31, 19, 33, 39, 0, time.UTC),
1840 EventBasedHold: false,
1841 Etag: "Zkyw9ACJZUvcYmlFaKGChzhmtnE/dt1zHSfweiWpwzdGsqXwuJZqiD0",
1842 Generation: 7,
1843 MD5: []byte("14683cba444dbcc6db297645e683f5c1"),
1844 Name: "foo.mp4",
1845 RetentionExpirationTime: time.Date(2019, 3, 31, 19, 33, 36, 0, time.UTC),
1846 Size: 1 << 20,
1847 TemporaryHold: true,
1848 }
1849 want := &raw.Object{
1850 Bucket: bucketName,
1851 ContentLanguage: "en-us",
1852 ContentType: "video/mpeg",
1853 EventBasedHold: false,
1854 Name: "foo.mp4",
1855 RetentionExpirationTime: "2019-03-31T19:33:36Z",
1856 TemporaryHold: true,
1857 }
1858 got := in.toRawObject(bucketName)
1859 if !testutil.Equal(got, want) {
1860 if diff := testutil.Diff(got, want); diff != "" {
1861 t.Errorf("toRawObject mismatches:\ngot=-, want=+:\n%s", diff)
1862 }
1863 }
1864 }
1865
1866 func TestProtoObjectToObjectAttrs(t *testing.T) {
1867 t.Parallel()
1868 now := time.Now()
1869 tests := []struct {
1870 in *storagepb.Object
1871 want *ObjectAttrs
1872 }{
1873 {in: nil, want: nil},
1874 {
1875 in: &storagepb.Object{
1876 Bucket: "Test",
1877 ContentLanguage: "en-us",
1878 ContentType: "video/mpeg",
1879 CustomTime: timestamppb.New(now),
1880 EventBasedHold: proto.Bool(false),
1881 Generation: 7,
1882 Checksums: &storagepb.ObjectChecksums{Md5Hash: []byte("14683cba444dbcc6db297645e683f5c1")},
1883 Name: "foo.mp4",
1884 RetentionExpireTime: timestamppb.New(now),
1885 Size: 1 << 20,
1886 CreateTime: timestamppb.New(now),
1887 DeleteTime: timestamppb.New(now),
1888 TemporaryHold: true,
1889 ComponentCount: 2,
1890 },
1891 want: &ObjectAttrs{
1892 Bucket: "Test",
1893 Created: now,
1894 ContentLanguage: "en-us",
1895 ContentType: "video/mpeg",
1896 CustomTime: now,
1897 Deleted: now,
1898 EventBasedHold: false,
1899 Generation: 7,
1900 MD5: []byte("14683cba444dbcc6db297645e683f5c1"),
1901 Name: "foo.mp4",
1902 RetentionExpirationTime: now,
1903 Size: 1 << 20,
1904 TemporaryHold: true,
1905 ComponentCount: 2,
1906 },
1907 },
1908 }
1909
1910 for i, tt := range tests {
1911 got := newObjectFromProto(tt.in)
1912 if diff := testutil.Diff(got, tt.want); diff != "" {
1913 t.Errorf("#%d: newObject mismatches:\ngot=-, want=+:\n%s", i, diff)
1914 }
1915 }
1916 }
1917
1918 func TestObjectAttrsToProtoObject(t *testing.T) {
1919 t.Parallel()
1920 now := time.Now()
1921
1922 b := "bucket"
1923 want := &storagepb.Object{
1924 Bucket: "projects/_/buckets/" + b,
1925 ContentLanguage: "en-us",
1926 ContentType: "video/mpeg",
1927 CustomTime: timestamppb.New(now),
1928 EventBasedHold: proto.Bool(false),
1929 Generation: 7,
1930 Name: "foo.mp4",
1931 RetentionExpireTime: timestamppb.New(now),
1932 Size: 1 << 20,
1933 CreateTime: timestamppb.New(now),
1934 DeleteTime: timestamppb.New(now),
1935 TemporaryHold: true,
1936 }
1937 in := &ObjectAttrs{
1938 Created: now,
1939 ContentLanguage: "en-us",
1940 ContentType: "video/mpeg",
1941 CustomTime: now,
1942 Deleted: now,
1943 EventBasedHold: false,
1944 Generation: 7,
1945 Name: "foo.mp4",
1946 RetentionExpirationTime: now,
1947 Size: 1 << 20,
1948 TemporaryHold: true,
1949 }
1950
1951 got := in.toProtoObject(b)
1952 if diff := testutil.Diff(got, want); diff != "" {
1953 t.Errorf("toProtoObject mismatches:\ngot=-, want=+:\n%s", diff)
1954 }
1955 }
1956
1957 func TestApplyCondsProto(t *testing.T) {
1958 for _, tst := range []struct {
1959 name string
1960 in, want proto.Message
1961 err error
1962 gen int64
1963 conds *Conditions
1964 }{
1965 {
1966 name: "generation",
1967 gen: 123,
1968 in: &storagepb.ReadObjectRequest{},
1969 want: &storagepb.ReadObjectRequest{Generation: 123},
1970 },
1971 {
1972 name: "invalid_no_generation",
1973 gen: 123,
1974 in: &storagepb.WriteObjectRequest{},
1975 err: fmt.Errorf("generation not supported"),
1976 },
1977 {
1978 name: "if_match",
1979 gen: -1,
1980 in: &storagepb.ReadObjectRequest{},
1981 want: &storagepb.ReadObjectRequest{IfGenerationMatch: proto.Int64(123), IfMetagenerationMatch: proto.Int64(123)},
1982 conds: &Conditions{GenerationMatch: 123, MetagenerationMatch: 123},
1983 },
1984 {
1985 name: "if_dne",
1986 gen: -1,
1987 in: &storagepb.ReadObjectRequest{},
1988 want: &storagepb.ReadObjectRequest{IfGenerationMatch: proto.Int64(0)},
1989 conds: &Conditions{DoesNotExist: true},
1990 },
1991 {
1992 name: "if_not_match",
1993 gen: -1,
1994 in: &storagepb.ReadObjectRequest{},
1995 want: &storagepb.ReadObjectRequest{IfGenerationNotMatch: proto.Int64(123), IfMetagenerationNotMatch: proto.Int64(123)},
1996 conds: &Conditions{GenerationNotMatch: 123, MetagenerationNotMatch: 123},
1997 },
1998 {
1999 name: "invalid_multiple_conditions",
2000 gen: -1,
2001 in: &storagepb.ReadObjectRequest{},
2002 conds: &Conditions{MetagenerationMatch: 123, MetagenerationNotMatch: 123},
2003 err: fmt.Errorf("multiple conditions"),
2004 },
2005 } {
2006 if err := applyCondsProto(tst.name, tst.gen, tst.conds, tst.in); tst.err == nil && err != nil {
2007 t.Errorf("%s: error got %v, want nil", tst.name, err)
2008 } else if tst.err != nil && (err == nil || !strings.Contains(err.Error(), tst.err.Error())) {
2009 t.Errorf("%s: error got %v, want %v", tst.name, err, tst.err)
2010 } else if diff := cmp.Diff(tst.in, tst.want, cmp.Comparer(proto.Equal)); tst.err == nil && diff != "" {
2011 t.Errorf("%s: got(-),want(+):\n%s", tst.name, diff)
2012 }
2013 }
2014 }
2015
2016 func TestAttrToFieldMapCoverage(t *testing.T) {
2017 t.Parallel()
2018
2019 oa := reflect.TypeOf((*ObjectAttrs)(nil)).Elem()
2020 oaFields := make(map[string]bool)
2021
2022 for i := 0; i < oa.NumField(); i++ {
2023 fieldName := oa.Field(i).Name
2024 oaFields[fieldName] = true
2025 }
2026
2027
2028 for k := range attrToFieldMap {
2029 if _, ok := oaFields[k]; !ok {
2030 t.Errorf("%v is not an ObjectAttrs field", k)
2031 }
2032 }
2033
2034
2035
2036
2037 for k := range oaFields {
2038 if _, ok := attrToFieldMap[k]; !ok {
2039 if k != "Prefix" && k != "PredefinedACL" {
2040 t.Errorf("ObjectAttrs.%v is not in attrToFieldMap", k)
2041 }
2042 }
2043 }
2044 }
2045
2046 func TestEmulatorWithCredentialsFile(t *testing.T) {
2047 t.Setenv("STORAGE_EMULATOR_HOST", "localhost:1234")
2048
2049 client, err := NewClient(context.Background(), option.WithCredentialsFile("/path/to/key.json"))
2050 if err != nil {
2051 t.Fatalf("failed creating a client with credentials file when running agains an emulator: %v", err)
2052 return
2053 }
2054 client.Close()
2055 }
2056
2057
2058
2059
2060 func TestWithEndpoint(t *testing.T) {
2061 originalStorageEmulatorHost := os.Getenv("STORAGE_EMULATOR_HOST")
2062 testCases := []struct {
2063 desc string
2064 CustomEndpoint string
2065 StorageEmulatorHost string
2066 WantRawBasePath string
2067 WantXMLHost string
2068 WantScheme string
2069 }{
2070 {
2071 desc: "No endpoint or emulator host specified",
2072 CustomEndpoint: "",
2073 StorageEmulatorHost: "",
2074 WantRawBasePath: "https://storage.googleapis.com/storage/v1/",
2075 WantXMLHost: "storage.googleapis.com",
2076 WantScheme: "https",
2077 },
2078 {
2079 desc: "With specified https endpoint, no specified emulator host",
2080 CustomEndpoint: "https://fake.gcs.com:8080/storage/v1",
2081 StorageEmulatorHost: "",
2082 WantRawBasePath: "https://fake.gcs.com:8080/storage/v1",
2083 WantXMLHost: "fake.gcs.com:8080",
2084 WantScheme: "https",
2085 },
2086 {
2087 desc: "With specified http endpoint, no specified emulator host",
2088 CustomEndpoint: "http://fake.gcs.com:8080/storage/v1",
2089 StorageEmulatorHost: "",
2090 WantRawBasePath: "http://fake.gcs.com:8080/storage/v1",
2091 WantXMLHost: "fake.gcs.com:8080",
2092 WantScheme: "http",
2093 },
2094 {
2095 desc: "Emulator host specified, no specified endpoint",
2096 CustomEndpoint: "",
2097 StorageEmulatorHost: "http://emu.com",
2098 WantRawBasePath: "http://emu.com/storage/v1/",
2099 WantXMLHost: "emu.com",
2100 WantScheme: "http",
2101 },
2102 {
2103 desc: "Emulator host specified without scheme",
2104 CustomEndpoint: "",
2105 StorageEmulatorHost: "emu.com",
2106 WantRawBasePath: "http://emu.com/storage/v1/",
2107 WantXMLHost: "emu.com",
2108 WantScheme: "http",
2109 },
2110 {
2111 desc: "Emulator host specified as host:port",
2112 CustomEndpoint: "",
2113 StorageEmulatorHost: "localhost:9000",
2114 WantRawBasePath: "http://localhost:9000/storage/v1/",
2115 WantXMLHost: "localhost:9000",
2116 WantScheme: "http",
2117 },
2118 {
2119 desc: "Endpoint overrides emulator host when both are specified - https",
2120 CustomEndpoint: "https://fake.gcs.com:8080/storage/v1",
2121 StorageEmulatorHost: "http://emu.com",
2122 WantRawBasePath: "https://fake.gcs.com:8080/storage/v1",
2123 WantXMLHost: "fake.gcs.com:8080",
2124 WantScheme: "https",
2125 },
2126 {
2127 desc: "Endpoint overrides emulator host when both are specified - http",
2128 CustomEndpoint: "http://fake.gcs.com:8080/storage/v1",
2129 StorageEmulatorHost: "https://emu.com",
2130 WantRawBasePath: "http://fake.gcs.com:8080/storage/v1",
2131 WantXMLHost: "fake.gcs.com:8080",
2132 WantScheme: "http",
2133 },
2134 {
2135 desc: "Endpoint overrides emulator host when host is specified as scheme://host:port",
2136 CustomEndpoint: "http://localhost:8080/storage/v1",
2137 StorageEmulatorHost: "https://localhost:9000",
2138 WantRawBasePath: "http://localhost:8080/storage/v1",
2139 WantXMLHost: "localhost:8080",
2140 WantScheme: "http",
2141 },
2142 {
2143 desc: "Endpoint overrides emulator host when host is specified as host:port",
2144 CustomEndpoint: "http://localhost:8080/storage/v1",
2145 StorageEmulatorHost: "localhost:9000",
2146 WantRawBasePath: "http://localhost:8080/storage/v1",
2147 WantXMLHost: "localhost:8080",
2148 WantScheme: "http",
2149 },
2150 }
2151 ctx := context.Background()
2152 for _, tc := range testCases {
2153 os.Setenv("STORAGE_EMULATOR_HOST", tc.StorageEmulatorHost)
2154 c, err := NewClient(ctx, option.WithEndpoint(tc.CustomEndpoint), option.WithoutAuthentication())
2155 if err != nil {
2156 t.Fatalf("error creating client: %v", err)
2157 }
2158
2159 if c.raw.BasePath != tc.WantRawBasePath {
2160 t.Errorf("%s: raw.BasePath not set correctly\n\tgot %v, want %v", tc.desc, c.raw.BasePath, tc.WantRawBasePath)
2161 }
2162 if c.xmlHost != tc.WantXMLHost {
2163 t.Errorf("%s: xmlHost not set correctly\n\tgot %v, want %v", tc.desc, c.xmlHost, tc.WantXMLHost)
2164 }
2165 if c.scheme != tc.WantScheme {
2166 t.Errorf("%s: scheme not set correctly\n\tgot %v, want %v", tc.desc, c.scheme, tc.WantScheme)
2167 }
2168 }
2169 os.Setenv("STORAGE_EMULATOR_HOST", originalStorageEmulatorHost)
2170 }
2171
2172
2173
2174
2175
2176
2177 func TestOperationsWithEndpoint(t *testing.T) {
2178 originalStorageEmulatorHost := os.Getenv("STORAGE_EMULATOR_HOST")
2179 defer os.Setenv("STORAGE_EMULATOR_HOST", originalStorageEmulatorHost)
2180
2181 gotURL := make(chan string, 1)
2182 gotHost := make(chan string, 1)
2183 gotMethod := make(chan string, 1)
2184
2185 timedOut := make(chan bool, 1)
2186
2187 hClient, closeServer := newTestServer(func(w http.ResponseWriter, r *http.Request) {
2188 done := make(chan bool, 1)
2189 io.Copy(ioutil.Discard, r.Body)
2190 fmt.Fprintf(w, "{}")
2191 go func() {
2192 gotHost <- r.Host
2193 gotURL <- r.RequestURI
2194 gotMethod <- r.Method
2195 done <- true
2196 }()
2197
2198 select {
2199 case <-timedOut:
2200 case <-done:
2201 }
2202
2203 })
2204 defer closeServer()
2205
2206 testCases := []struct {
2207 desc string
2208 CustomEndpoint string
2209 StorageEmulatorHost string
2210 wantScheme string
2211 wantHost string
2212 }{
2213 {
2214 desc: "No endpoint or emulator host specified",
2215 CustomEndpoint: "",
2216 StorageEmulatorHost: "",
2217 wantScheme: "https",
2218 wantHost: "storage.googleapis.com",
2219 },
2220 {
2221 desc: "emulator host specified",
2222 CustomEndpoint: "",
2223 StorageEmulatorHost: "https://" + "addr",
2224 wantScheme: "https",
2225 wantHost: "addr",
2226 },
2227 {
2228 desc: "endpoint specified",
2229 CustomEndpoint: "https://" + "end" + "/storage/v1/",
2230 StorageEmulatorHost: "",
2231 wantScheme: "https",
2232 wantHost: "end",
2233 },
2234 {
2235 desc: "both emulator and endpoint specified",
2236 CustomEndpoint: "https://" + "end" + "/storage/v1/",
2237 StorageEmulatorHost: "http://host",
2238 wantScheme: "https",
2239 wantHost: "end",
2240 },
2241 }
2242
2243 for _, tc := range testCases {
2244 ctx := context.Background()
2245 t.Run(tc.desc, func(t *testing.T) {
2246 timeout := time.After(time.Second)
2247 done := make(chan bool, 1)
2248 go func() {
2249 os.Setenv("STORAGE_EMULATOR_HOST", tc.StorageEmulatorHost)
2250
2251 c, err := NewClient(ctx, option.WithHTTPClient(hClient), option.WithEndpoint(tc.CustomEndpoint))
2252 if err != nil {
2253 t.Errorf("error creating client: %v", err)
2254 return
2255 }
2256 originalRawBasePath := c.raw.BasePath
2257 originalXMLHost := c.xmlHost
2258 originalScheme := c.scheme
2259
2260 operations := []struct {
2261 desc string
2262 runOp func() error
2263 wantURL string
2264 wantMethod string
2265 }{
2266 {
2267 desc: "Create a bucket",
2268 runOp: func() error {
2269 return c.Bucket("test-bucket").Create(ctx, "pid", nil)
2270 },
2271 wantURL: "/storage/v1/b?alt=json&prettyPrint=false&project=pid",
2272 wantMethod: "POST",
2273 },
2274 {
2275 desc: "Upload an object",
2276 runOp: func() error {
2277 w := c.Bucket("test-bucket").Object("file").NewWriter(ctx)
2278 _, err = io.Copy(w, strings.NewReader("copyng into bucket"))
2279 if err != nil {
2280 return err
2281 }
2282 return w.Close()
2283 },
2284 wantURL: "/upload/storage/v1/b/test-bucket/o?alt=json&name=file&prettyPrint=false&projection=full&uploadType=multipart",
2285 wantMethod: "POST",
2286 },
2287 {
2288 desc: "Download an object",
2289 runOp: func() error {
2290 rc, err := c.Bucket("test-bucket").Object("file").NewReader(ctx)
2291 if err != nil {
2292 return err
2293 }
2294
2295 _, err = io.Copy(ioutil.Discard, rc)
2296 if err != nil {
2297 return err
2298 }
2299 return rc.Close()
2300 },
2301 wantURL: "/test-bucket/file",
2302 wantMethod: "GET",
2303 },
2304 {
2305 desc: "Delete bucket",
2306 runOp: func() error {
2307 return c.Bucket("test-bucket").Delete(ctx)
2308 },
2309 wantURL: "/storage/v1/b/test-bucket?alt=json&prettyPrint=false",
2310 wantMethod: "DELETE",
2311 },
2312 }
2313
2314
2315
2316 for _, op := range operations {
2317 if err := op.runOp(); err != nil {
2318 t.Errorf("%s: %v", op.desc, err)
2319 }
2320 u, method := <-gotURL, <-gotMethod
2321 if u != op.wantURL {
2322 t.Errorf("%s: unexpected request URL\ngot %q\nwant %q",
2323 op.desc, u, op.wantURL)
2324 }
2325 if method != op.wantMethod {
2326 t.Errorf("%s: unexpected request method\ngot %q\nwant %q",
2327 op.desc, method, op.wantMethod)
2328 }
2329
2330 if got := <-gotHost; got != tc.wantHost {
2331 t.Errorf("%s: unexpected request host\ngot %q\nwant %q",
2332 op.desc, got, tc.wantHost)
2333 }
2334 }
2335
2336
2337 if c.raw.BasePath != originalRawBasePath {
2338 t.Errorf("raw.BasePath changed\n\tgot:\t\t%v\n\toriginal:\t%v",
2339 c.raw.BasePath, originalRawBasePath)
2340 }
2341 if c.xmlHost != originalXMLHost {
2342 t.Errorf("xmlHost changed\n\tgot:\t\t%v\n\toriginal:\t%v",
2343 c.xmlHost, originalXMLHost)
2344 }
2345 if c.scheme != originalScheme {
2346 t.Errorf("scheme changed\n\tgot:\t\t%v\n\toriginal:\t%v",
2347 c.scheme, originalScheme)
2348 }
2349 done <- true
2350 }()
2351 select {
2352 case <-timeout:
2353 t.Errorf("test timeout")
2354 timedOut <- true
2355 case <-done:
2356 }
2357 })
2358
2359 }
2360 }
2361
2362 func TestSignedURLOptionsClone(t *testing.T) {
2363 t.Parallel()
2364
2365 opts := &SignedURLOptions{
2366 GoogleAccessID: "accessID",
2367 PrivateKey: []byte{},
2368 SignBytes: func(b []byte) ([]byte, error) {
2369 return b, nil
2370 },
2371 Method: "GET",
2372 Expires: time.Now(),
2373 ContentType: "text/plain",
2374 Headers: []string{},
2375 QueryParameters: map[string][]string{},
2376 MD5: "some-checksum",
2377 Style: VirtualHostedStyle(),
2378 Insecure: true,
2379 Scheme: SigningSchemeV2,
2380 Hostname: "localhost:8000",
2381 }
2382
2383
2384
2385 reflectOpts := reflect.ValueOf(*opts)
2386 for i := 0; i < reflectOpts.NumField(); i++ {
2387 zero, err := isZeroValue(reflectOpts.Field(i))
2388 if err != nil {
2389 t.Errorf("IsZero: %v", err)
2390 }
2391 if zero {
2392 t.Errorf("SignedURLOptions field %d not set", i)
2393 }
2394 }
2395
2396
2397 optsClone := opts.clone()
2398
2399
2400 signBytesComp := func(a func([]byte) ([]byte, error), b func([]byte) ([]byte, error)) bool {
2401 return reflect.ValueOf(a) == reflect.ValueOf(b)
2402 }
2403
2404 if diff := cmp.Diff(opts, optsClone, cmp.Comparer(signBytesComp), cmp.AllowUnexported(SignedURLOptions{})); diff != "" {
2405 t.Errorf("clone does not match (original: -, cloned: +):\n%s", diff)
2406 }
2407 }
2408
2409 func TestParseProjectNumber(t *testing.T) {
2410 for _, tst := range []struct {
2411 input string
2412 want uint64
2413 }{
2414 {"projects/123", 123},
2415 {"projects/123/foos/456", 123},
2416 {"projects/abc-123/foos/456", 0},
2417 {"projects/abc-123", 0},
2418 {"projects/abc", 0},
2419 {"projects/abc/foos", 0},
2420 } {
2421 if got := parseProjectNumber(tst.input); got != tst.want {
2422 t.Errorf("For %q: got %v, expected %v", tst.input, got, tst.want)
2423 }
2424 }
2425 }
2426
2427 func TestObjectValidate(t *testing.T) {
2428 for _, c := range []struct {
2429 name string
2430 bucket string
2431 object string
2432 wantSuccess bool
2433 }{
2434 {
2435 name: "valid object",
2436 bucket: "my-bucket",
2437 object: "my-object",
2438 wantSuccess: true,
2439 },
2440 {
2441 name: "empty bucket name",
2442 bucket: "",
2443 object: "my-object",
2444 wantSuccess: false,
2445 },
2446 {
2447 name: "empty object name",
2448 bucket: "my-bucket",
2449 object: "",
2450 wantSuccess: false,
2451 },
2452 {
2453 name: "invalid utf-8",
2454 bucket: "my-bucket",
2455 object: "\xc3\x28",
2456 wantSuccess: false,
2457 },
2458 {
2459 name: "object name .",
2460 bucket: "my-bucket",
2461 object: ".",
2462 wantSuccess: false,
2463 },
2464 } {
2465 t.Run(c.name, func(r *testing.T) {
2466 b := &BucketHandle{name: c.bucket}
2467 err := b.Object(c.object).validate()
2468 if c.wantSuccess && err != nil {
2469 r.Errorf("want success, got error %v", err)
2470 }
2471 if !c.wantSuccess && err == nil {
2472 r.Errorf("want error, got nil")
2473 }
2474 })
2475 }
2476 }
2477
2478
2479
2480 func isZeroValue(v reflect.Value) (bool, error) {
2481 switch v.Kind() {
2482 case reflect.Bool:
2483 return !v.Bool(), nil
2484 case reflect.Int, reflect.Int64:
2485 return v.Int() == 0, nil
2486 case reflect.Uint, reflect.Uint64:
2487 return v.Uint() == 0, nil
2488 case reflect.Array:
2489 for i := 0; i < v.Len(); i++ {
2490 zero, err := isZeroValue(v.Index(i))
2491 if !zero || err != nil {
2492 return false, err
2493 }
2494 }
2495 return true, nil
2496 case reflect.Func, reflect.Interface, reflect.Map, reflect.Slice, reflect.Ptr:
2497 return v.IsNil(), nil
2498 case reflect.String:
2499 return v.Len() == 0, nil
2500 case reflect.Struct:
2501 for i := 0; i < v.NumField(); i++ {
2502 zero, err := isZeroValue(v.Field(i))
2503 if !zero || err != nil {
2504 return false, err
2505 }
2506 }
2507 return true, nil
2508 default:
2509 return false, fmt.Errorf("unable to check kind %s", v.Kind())
2510 }
2511 }
2512
View as plain text