1#!/usr/bin/env python 2# 3# Copyright 2014 Google Inc. All Rights Reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Http tests 18 19Unit tests for the googleapiclient.http. 20""" 21from __future__ import absolute_import 22 23__author__ = "jcgregorio@google.com (Joe Gregorio)" 24 25from io import FileIO 26 27# Do not remove the httplib2 import 28import json 29import httplib2 30import io 31import logging 32import mock 33import os 34import unittest 35import urllib 36import random 37import socket 38import ssl 39import time 40 41from googleapiclient.discovery import build 42from googleapiclient.errors import BatchError 43from googleapiclient.errors import HttpError 44from googleapiclient.errors import InvalidChunkSizeError 45from googleapiclient.http import build_http 46from googleapiclient.http import BatchHttpRequest 47from googleapiclient.http import HttpMock 48from googleapiclient.http import HttpMockSequence 49from googleapiclient.http import HttpRequest 50from googleapiclient.http import MAX_URI_LENGTH 51from googleapiclient.http import MediaFileUpload 52from googleapiclient.http import MediaInMemoryUpload 53from googleapiclient.http import MediaIoBaseDownload 54from googleapiclient.http import MediaIoBaseUpload 55from googleapiclient.http import MediaUpload 56from googleapiclient.http import _StreamSlice 57from googleapiclient.http import set_user_agent 58from googleapiclient.model import JsonModel 59from oauth2client.client import Credentials 60 61 62class MockCredentials(Credentials): 63 """Mock class for all Credentials objects.""" 64 65 def __init__(self, bearer_token, expired=False): 66 super(MockCredentials, self).__init__() 67 self._authorized = 0 68 self._refreshed = 0 69 self._applied = 0 70 self._bearer_token = bearer_token 71 self._access_token_expired = expired 72 73 @property 74 def access_token(self): 75 return self._bearer_token 76 77 @property 78 def access_token_expired(self): 79 return self._access_token_expired 80 81 def authorize(self, http): 82 self._authorized += 1 83 84 request_orig = http.request 85 86 # The closure that will replace 'httplib2.Http.request'. 87 def new_request( 88 uri, 89 method="GET", 90 body=None, 91 headers=None, 92 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 93 connection_type=None, 94 ): 95 # Modify the request headers to add the appropriate 96 # Authorization header. 97 if headers is None: 98 headers = {} 99 self.apply(headers) 100 101 resp, content = request_orig( 102 uri, method, body, headers, redirections, connection_type 103 ) 104 105 return resp, content 106 107 # Replace the request method with our own closure. 108 http.request = new_request 109 110 # Set credentials as a property of the request method. 111 setattr(http.request, "credentials", self) 112 113 return http 114 115 def refresh(self, http): 116 self._refreshed += 1 117 118 def apply(self, headers): 119 self._applied += 1 120 headers["authorization"] = self._bearer_token + " " + str(self._refreshed) 121 122 123class HttpMockWithErrors(object): 124 def __init__(self, num_errors, success_json, success_data): 125 self.num_errors = num_errors 126 self.success_json = success_json 127 self.success_data = success_data 128 129 def request(self, *args, **kwargs): 130 if not self.num_errors: 131 return httplib2.Response(self.success_json), self.success_data 132 elif self.num_errors == 5: 133 ex = ConnectionResetError # noqa: F821 134 elif self.num_errors == 4: 135 ex = httplib2.ServerNotFoundError() 136 elif self.num_errors == 3: 137 ex = OSError() 138 ex.errno = socket.errno.EPIPE 139 elif self.num_errors == 2: 140 ex = ssl.SSLError() 141 else: 142 # Initialize the timeout error code to the platform's error code. 143 try: 144 # For Windows: 145 ex = OSError() 146 ex.errno = socket.errno.WSAETIMEDOUT 147 except AttributeError: 148 # For Linux/Mac: 149 ex = socket.timeout() 150 151 self.num_errors -= 1 152 raise ex 153 154 155class HttpMockWithNonRetriableErrors(object): 156 def __init__(self, num_errors, success_json, success_data): 157 self.num_errors = num_errors 158 self.success_json = success_json 159 self.success_data = success_data 160 161 def request(self, *args, **kwargs): 162 if not self.num_errors: 163 return httplib2.Response(self.success_json), self.success_data 164 else: 165 self.num_errors -= 1 166 ex = OSError() 167 # set errno to a non-retriable value 168 try: 169 # For Windows: 170 ex.errno = socket.errno.WSAEHOSTUNREACH 171 except AttributeError: 172 # For Linux/Mac: 173 ex.errno = socket.errno.EHOSTUNREACH 174 # Now raise the correct timeout error. 175 raise ex 176 177 178DATA_DIR = os.path.join(os.path.dirname(__file__), "data") 179 180 181def datafile(filename): 182 return os.path.join(DATA_DIR, filename) 183 184 185def _postproc_none(*kwargs): 186 pass 187 188 189class TestUserAgent(unittest.TestCase): 190 def test_set_user_agent(self): 191 http = HttpMockSequence([({"status": "200"}, "echo_request_headers")]) 192 193 http = set_user_agent(http, "my_app/5.5") 194 resp, content = http.request("http://example.com") 195 self.assertEqual("my_app/5.5", content["user-agent"]) 196 197 def test_set_user_agent_nested(self): 198 http = HttpMockSequence([({"status": "200"}, "echo_request_headers")]) 199 200 http = set_user_agent(http, "my_app/5.5") 201 http = set_user_agent(http, "my_library/0.1") 202 resp, content = http.request("http://example.com") 203 self.assertEqual("my_app/5.5 my_library/0.1", content["user-agent"]) 204 205 206class TestMediaUpload(unittest.TestCase): 207 def test_media_file_upload_closes_fd_in___del__(self): 208 file_desc = mock.Mock(spec=io.TextIOWrapper) 209 opener = mock.mock_open(file_desc) 210 with mock.patch("builtins.open", return_value=opener): 211 upload = MediaFileUpload(datafile("test_close"), mimetype="text/plain") 212 self.assertIs(upload.stream(), file_desc) 213 del upload 214 file_desc.close.assert_called_once_with() 215 216 def test_media_file_upload_mimetype_detection(self): 217 upload = MediaFileUpload(datafile("small.png")) 218 self.assertEqual("image/png", upload.mimetype()) 219 220 upload = MediaFileUpload(datafile("empty")) 221 self.assertEqual("application/octet-stream", upload.mimetype()) 222 223 def test_media_file_upload_to_from_json(self): 224 upload = MediaFileUpload(datafile("small.png"), chunksize=500, resumable=True) 225 self.assertEqual("image/png", upload.mimetype()) 226 self.assertEqual(190, upload.size()) 227 self.assertEqual(True, upload.resumable()) 228 self.assertEqual(500, upload.chunksize()) 229 self.assertEqual(b"PNG", upload.getbytes(1, 3)) 230 231 json = upload.to_json() 232 new_upload = MediaUpload.new_from_json(json) 233 234 self.assertEqual("image/png", new_upload.mimetype()) 235 self.assertEqual(190, new_upload.size()) 236 self.assertEqual(True, new_upload.resumable()) 237 self.assertEqual(500, new_upload.chunksize()) 238 self.assertEqual(b"PNG", new_upload.getbytes(1, 3)) 239 240 def test_media_file_upload_raises_on_file_not_found(self): 241 with self.assertRaises(FileNotFoundError): 242 MediaFileUpload(datafile("missing.png")) 243 244 def test_media_file_upload_raises_on_invalid_chunksize(self): 245 self.assertRaises( 246 InvalidChunkSizeError, 247 MediaFileUpload, 248 datafile("small.png"), 249 mimetype="image/png", 250 chunksize=-2, 251 resumable=True, 252 ) 253 254 def test_media_inmemory_upload(self): 255 media = MediaInMemoryUpload( 256 b"abcdef", mimetype="text/plain", chunksize=10, resumable=True 257 ) 258 self.assertEqual("text/plain", media.mimetype()) 259 self.assertEqual(10, media.chunksize()) 260 self.assertTrue(media.resumable()) 261 self.assertEqual(b"bc", media.getbytes(1, 2)) 262 self.assertEqual(6, media.size()) 263 264 def test_http_request_to_from_json(self): 265 http = build_http() 266 media_upload = MediaFileUpload( 267 datafile("small.png"), chunksize=500, resumable=True 268 ) 269 req = HttpRequest( 270 http, 271 _postproc_none, 272 "http://example.com", 273 method="POST", 274 body="{}", 275 headers={"content-type": 'multipart/related; boundary="---flubber"'}, 276 methodId="foo", 277 resumable=media_upload, 278 ) 279 280 json = req.to_json() 281 new_req = HttpRequest.from_json(json, http, _postproc_none) 282 283 self.assertEqual( 284 {"content-type": 'multipart/related; boundary="---flubber"'}, 285 new_req.headers, 286 ) 287 self.assertEqual("http://example.com", new_req.uri) 288 self.assertEqual("{}", new_req.body) 289 self.assertEqual(http, new_req.http) 290 self.assertEqual(media_upload.to_json(), new_req.resumable.to_json()) 291 292 self.assertEqual(random.random, new_req._rand) 293 self.assertEqual(time.sleep, new_req._sleep) 294 295 296class TestMediaIoBaseUpload(unittest.TestCase): 297 def test_media_io_base_upload_from_file_io(self): 298 fd = FileIO(datafile("small.png"), "r") 299 upload = MediaIoBaseUpload( 300 fd=fd, mimetype="image/png", chunksize=500, resumable=True 301 ) 302 self.assertEqual("image/png", upload.mimetype()) 303 self.assertEqual(190, upload.size()) 304 self.assertEqual(True, upload.resumable()) 305 self.assertEqual(500, upload.chunksize()) 306 self.assertEqual(b"PNG", upload.getbytes(1, 3)) 307 308 def test_media_io_base_upload_from_file_object(self): 309 f = open(datafile("small.png"), "rb") 310 upload = MediaIoBaseUpload( 311 fd=f, mimetype="image/png", chunksize=500, resumable=True 312 ) 313 self.assertEqual("image/png", upload.mimetype()) 314 self.assertEqual(190, upload.size()) 315 self.assertEqual(True, upload.resumable()) 316 self.assertEqual(500, upload.chunksize()) 317 self.assertEqual(b"PNG", upload.getbytes(1, 3)) 318 f.close() 319 320 def test_media_io_base_upload_serializable(self): 321 f = open(datafile("small.png"), "rb") 322 upload = MediaIoBaseUpload(fd=f, mimetype="image/png") 323 324 try: 325 json = upload.to_json() 326 self.fail("MediaIoBaseUpload should not be serializable.") 327 except NotImplementedError: 328 pass 329 330 331 def test_media_io_base_upload_from_bytes(self): 332 f = open(datafile("small.png"), "rb") 333 fd = io.BytesIO(f.read()) 334 upload = MediaIoBaseUpload( 335 fd=fd, mimetype="image/png", chunksize=500, resumable=True 336 ) 337 self.assertEqual("image/png", upload.mimetype()) 338 self.assertEqual(190, upload.size()) 339 self.assertEqual(True, upload.resumable()) 340 self.assertEqual(500, upload.chunksize()) 341 self.assertEqual(b"PNG", upload.getbytes(1, 3)) 342 343 def test_media_io_base_upload_raises_on_invalid_chunksize(self): 344 f = open(datafile("small.png"), "rb") 345 fd = io.BytesIO(f.read()) 346 self.assertRaises( 347 InvalidChunkSizeError, 348 MediaIoBaseUpload, 349 fd, 350 "image/png", 351 chunksize=-2, 352 resumable=True, 353 ) 354 355 def test_media_io_base_upload_streamable(self): 356 fd = io.BytesIO(b"stuff") 357 upload = MediaIoBaseUpload( 358 fd=fd, mimetype="image/png", chunksize=500, resumable=True 359 ) 360 self.assertEqual(True, upload.has_stream()) 361 self.assertEqual(fd, upload.stream()) 362 363 def test_media_io_base_next_chunk_retries(self): 364 f = open(datafile("small.png"), "rb") 365 fd = io.BytesIO(f.read()) 366 upload = MediaIoBaseUpload( 367 fd=fd, mimetype="image/png", chunksize=500, resumable=True 368 ) 369 370 # Simulate errors for both the request that creates the resumable upload 371 # and the upload itself. 372 http = HttpMockSequence( 373 [ 374 ({"status": "500"}, ""), 375 ({"status": "500"}, ""), 376 ({"status": "503"}, ""), 377 ({"status": "200", "location": "location"}, ""), 378 ({"status": "403"}, USER_RATE_LIMIT_EXCEEDED_RESPONSE_NO_STATUS), 379 ({"status": "403"}, RATE_LIMIT_EXCEEDED_RESPONSE), 380 ({"status": "429"}, ""), 381 ({"status": "200"}, "{}"), 382 ] 383 ) 384 385 model = JsonModel() 386 uri = u"https://www.googleapis.com/someapi/v1/upload/?foo=bar" 387 method = u"POST" 388 request = HttpRequest( 389 http, model.response, uri, method=method, headers={}, resumable=upload 390 ) 391 392 sleeptimes = [] 393 request._sleep = lambda x: sleeptimes.append(x) 394 request._rand = lambda: 10 395 396 request.execute(num_retries=3) 397 self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes) 398 399 def test_media_io_base_next_chunk_no_retry_403_not_configured(self): 400 fd = io.BytesIO(b"i am png") 401 upload = MediaIoBaseUpload( 402 fd=fd, mimetype="image/png", chunksize=500, resumable=True 403 ) 404 405 http = HttpMockSequence( 406 [({"status": "403"}, NOT_CONFIGURED_RESPONSE), ({"status": "200"}, "{}")] 407 ) 408 409 model = JsonModel() 410 uri = u"https://www.googleapis.com/someapi/v1/upload/?foo=bar" 411 method = u"POST" 412 request = HttpRequest( 413 http, model.response, uri, method=method, headers={}, resumable=upload 414 ) 415 416 request._rand = lambda: 1.0 417 request._sleep = mock.MagicMock() 418 419 with self.assertRaises(HttpError): 420 request.execute(num_retries=3) 421 request._sleep.assert_not_called() 422 423 424 def test_media_io_base_empty_file(self): 425 fd = io.BytesIO() 426 upload = MediaIoBaseUpload( 427 fd=fd, mimetype="image/png", chunksize=500, resumable=True 428 ) 429 430 http = HttpMockSequence( 431 [ 432 ({"status": "200", "location": "https://www.googleapis.com/someapi/v1/upload?foo=bar"}, "{}"), 433 ({"status": "200", "location": "https://www.googleapis.com/someapi/v1/upload?foo=bar"}, "{}") 434 ] 435 ) 436 437 model = JsonModel() 438 uri = u"https://www.googleapis.com/someapi/v1/upload/?foo=bar" 439 method = u"POST" 440 request = HttpRequest( 441 http, model.response, uri, method=method, headers={}, resumable=upload 442 ) 443 444 request.execute() 445 446 # Check that "Content-Range" header is not set in the PUT request 447 self.assertTrue("Content-Range" not in http.request_sequence[-1][-1]) 448 self.assertEqual("0", http.request_sequence[-1][-1]["Content-Length"]) 449 450 451class TestMediaIoBaseDownload(unittest.TestCase): 452 def setUp(self): 453 http = HttpMock(datafile("zoo.json"), {"status": "200"}) 454 zoo = build("zoo", "v1", http=http, static_discovery=False) 455 self.request = zoo.animals().get_media(name="Lion") 456 self.fd = io.BytesIO() 457 458 def test_media_io_base_download(self): 459 self.request.http = HttpMockSequence( 460 [ 461 ({"status": "200", "content-range": "0-2/5"}, b"123"), 462 ({"status": "200", "content-range": "3-4/5"}, b"45"), 463 ] 464 ) 465 self.assertEqual(True, self.request.http.follow_redirects) 466 467 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 468 469 self.assertEqual(self.fd, download._fd) 470 self.assertEqual(3, download._chunksize) 471 self.assertEqual(0, download._progress) 472 self.assertEqual(None, download._total_size) 473 self.assertEqual(False, download._done) 474 self.assertEqual(self.request.uri, download._uri) 475 476 status, done = download.next_chunk() 477 478 self.assertEqual(self.fd.getvalue(), b"123") 479 self.assertEqual(False, done) 480 self.assertEqual(3, download._progress) 481 self.assertEqual(5, download._total_size) 482 self.assertEqual(3, status.resumable_progress) 483 484 status, done = download.next_chunk() 485 486 self.assertEqual(self.fd.getvalue(), b"12345") 487 self.assertEqual(True, done) 488 self.assertEqual(5, download._progress) 489 self.assertEqual(5, download._total_size) 490 491 def test_media_io_base_download_range_request_header(self): 492 self.request.http = HttpMockSequence( 493 [ 494 ( 495 {"status": "200", "content-range": "0-2/5"}, 496 "echo_request_headers_as_json", 497 ), 498 ] 499 ) 500 501 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 502 503 status, done = download.next_chunk() 504 result = json.loads(self.fd.getvalue().decode("utf-8")) 505 506 self.assertEqual(result.get("range"), "bytes=0-2") 507 508 def test_media_io_base_download_custom_request_headers(self): 509 self.request.http = HttpMockSequence( 510 [ 511 ( 512 {"status": "200", "content-range": "0-2/5"}, 513 "echo_request_headers_as_json", 514 ), 515 ( 516 {"status": "200", "content-range": "3-4/5"}, 517 "echo_request_headers_as_json", 518 ), 519 ] 520 ) 521 self.assertEqual(True, self.request.http.follow_redirects) 522 523 self.request.headers["Cache-Control"] = "no-store" 524 525 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 526 527 self.assertEqual(download._headers.get("Cache-Control"), "no-store") 528 529 status, done = download.next_chunk() 530 531 result = json.loads(self.fd.getvalue().decode("utf-8")) 532 533 # assert that that the header we added to the original request is 534 # sent up to the server on each call to next_chunk 535 536 self.assertEqual(result.get("Cache-Control"), "no-store") 537 538 download._fd = self.fd = io.BytesIO() 539 status, done = download.next_chunk() 540 541 result = json.loads(self.fd.getvalue().decode("utf-8")) 542 self.assertEqual(result.get("Cache-Control"), "no-store") 543 544 def test_media_io_base_download_handle_redirects(self): 545 self.request.http = HttpMockSequence( 546 [ 547 ( 548 { 549 "status": "200", 550 "content-location": "https://secure.example.net/lion", 551 }, 552 b"", 553 ), 554 ({"status": "200", "content-range": "0-2/5"}, b"abc"), 555 ] 556 ) 557 558 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 559 560 status, done = download.next_chunk() 561 562 self.assertEqual("https://secure.example.net/lion", download._uri) 563 564 def test_media_io_base_download_handle_4xx(self): 565 self.request.http = HttpMockSequence([({"status": "400"}, "")]) 566 567 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 568 569 try: 570 status, done = download.next_chunk() 571 self.fail("Should raise an exception") 572 except HttpError: 573 pass 574 575 # Even after raising an exception we can pick up where we left off. 576 self.request.http = HttpMockSequence( 577 [({"status": "200", "content-range": "0-2/5"}, b"123")] 578 ) 579 580 status, done = download.next_chunk() 581 582 self.assertEqual(self.fd.getvalue(), b"123") 583 584 def test_media_io_base_download_retries_connection_errors(self): 585 self.request.http = HttpMockWithErrors( 586 5, {"status": "200", "content-range": "0-2/3"}, b"123" 587 ) 588 589 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 590 download._sleep = lambda _x: 0 # do nothing 591 download._rand = lambda: 10 592 593 status, done = download.next_chunk(num_retries=5) 594 595 self.assertEqual(self.fd.getvalue(), b"123") 596 self.assertEqual(True, done) 597 598 def test_media_io_base_download_retries_5xx(self): 599 self.request.http = HttpMockSequence( 600 [ 601 ({"status": "500"}, ""), 602 ({"status": "500"}, ""), 603 ({"status": "500"}, ""), 604 ({"status": "200", "content-range": "0-2/5"}, b"123"), 605 ({"status": "503"}, ""), 606 ({"status": "503"}, ""), 607 ({"status": "503"}, ""), 608 ({"status": "200", "content-range": "3-4/5"}, b"45"), 609 ] 610 ) 611 612 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 613 614 self.assertEqual(self.fd, download._fd) 615 self.assertEqual(3, download._chunksize) 616 self.assertEqual(0, download._progress) 617 self.assertEqual(None, download._total_size) 618 self.assertEqual(False, download._done) 619 self.assertEqual(self.request.uri, download._uri) 620 621 # Set time.sleep and random.random stubs. 622 sleeptimes = [] 623 download._sleep = lambda x: sleeptimes.append(x) 624 download._rand = lambda: 10 625 626 status, done = download.next_chunk(num_retries=3) 627 628 # Check for exponential backoff using the rand function above. 629 self.assertEqual([20, 40, 80], sleeptimes) 630 631 self.assertEqual(self.fd.getvalue(), b"123") 632 self.assertEqual(False, done) 633 self.assertEqual(3, download._progress) 634 self.assertEqual(5, download._total_size) 635 self.assertEqual(3, status.resumable_progress) 636 637 # Reset time.sleep stub. 638 del sleeptimes[0 : len(sleeptimes)] 639 640 status, done = download.next_chunk(num_retries=3) 641 642 # Check for exponential backoff using the rand function above. 643 self.assertEqual([20, 40, 80], sleeptimes) 644 645 self.assertEqual(self.fd.getvalue(), b"12345") 646 self.assertEqual(True, done) 647 self.assertEqual(5, download._progress) 648 self.assertEqual(5, download._total_size) 649 650 def test_media_io_base_download_empty_file(self): 651 self.request.http = HttpMockSequence( 652 [({"status": "200", "content-range": "0-0/0"}, b"")] 653 ) 654 655 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 656 657 self.assertEqual(self.fd, download._fd) 658 self.assertEqual(0, download._progress) 659 self.assertEqual(None, download._total_size) 660 self.assertEqual(False, download._done) 661 self.assertEqual(self.request.uri, download._uri) 662 663 status, done = download.next_chunk() 664 665 self.assertEqual(True, done) 666 self.assertEqual(0, download._progress) 667 self.assertEqual(0, download._total_size) 668 self.assertEqual(0, status.progress()) 669 670 def test_media_io_base_download_empty_file_416_response(self): 671 self.request.http = HttpMockSequence( 672 [({"status": "416", "content-range": "0-0/0"}, b"")] 673 ) 674 675 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 676 677 self.assertEqual(self.fd, download._fd) 678 self.assertEqual(0, download._progress) 679 self.assertEqual(None, download._total_size) 680 self.assertEqual(False, download._done) 681 self.assertEqual(self.request.uri, download._uri) 682 683 status, done = download.next_chunk() 684 685 self.assertEqual(True, done) 686 self.assertEqual(0, download._progress) 687 self.assertEqual(0, download._total_size) 688 self.assertEqual(0, status.progress()) 689 690 def test_media_io_base_download_unknown_media_size(self): 691 self.request.http = HttpMockSequence([({"status": "200"}, b"123")]) 692 693 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3) 694 695 self.assertEqual(self.fd, download._fd) 696 self.assertEqual(0, download._progress) 697 self.assertEqual(None, download._total_size) 698 self.assertEqual(False, download._done) 699 self.assertEqual(self.request.uri, download._uri) 700 701 status, done = download.next_chunk() 702 703 self.assertEqual(self.fd.getvalue(), b"123") 704 self.assertEqual(True, done) 705 self.assertEqual(3, download._progress) 706 self.assertEqual(None, download._total_size) 707 self.assertEqual(0, status.progress()) 708 709 710EXPECTED = """POST /someapi/v1/collection/?foo=bar HTTP/1.1 711Content-Type: application/json 712MIME-Version: 1.0 713Host: www.googleapis.com 714content-length: 2\r\n\r\n{}""" 715 716 717NO_BODY_EXPECTED = """POST /someapi/v1/collection/?foo=bar HTTP/1.1 718Content-Type: application/json 719MIME-Version: 1.0 720Host: www.googleapis.com 721content-length: 0\r\n\r\n""" 722 723NO_BODY_EXPECTED_GET = """GET /someapi/v1/collection/?foo=bar HTTP/1.1 724Content-Type: application/json 725MIME-Version: 1.0 726Host: www.googleapis.com\r\n\r\n""" 727 728 729RESPONSE = """HTTP/1.1 200 OK 730Content-Type: application/json 731Content-Length: 14 732ETag: "etag/pony"\r\n\r\n{"answer": 42}""" 733 734 735BATCH_RESPONSE = b"""--batch_foobarbaz 736Content-Type: application/http 737Content-Transfer-Encoding: binary 738Content-ID: <randomness + 1> 739 740HTTP/1.1 200 OK 741Content-Type: application/json 742Content-Length: 14 743ETag: "etag/pony"\r\n\r\n{"foo": 42} 744 745--batch_foobarbaz 746Content-Type: application/http 747Content-Transfer-Encoding: binary 748Content-ID: <randomness + 2> 749 750HTTP/1.1 200 OK 751Content-Type: application/json 752Content-Length: 14 753ETag: "etag/sheep"\r\n\r\n{"baz": "qux"} 754--batch_foobarbaz--""" 755 756 757BATCH_ERROR_RESPONSE = b"""--batch_foobarbaz 758Content-Type: application/http 759Content-Transfer-Encoding: binary 760Content-ID: <randomness + 1> 761 762HTTP/1.1 200 OK 763Content-Type: application/json 764Content-Length: 14 765ETag: "etag/pony"\r\n\r\n{"foo": 42} 766 767--batch_foobarbaz 768Content-Type: application/http 769Content-Transfer-Encoding: binary 770Content-ID: <randomness + 2> 771 772HTTP/1.1 403 Access Not Configured 773Content-Type: application/json 774Content-Length: 245 775ETag: "etag/sheep"\r\n\r\n{ 776 "error": { 777 "errors": [ 778 { 779 "domain": "usageLimits", 780 "reason": "accessNotConfigured", 781 "message": "Access Not Configured", 782 "debugInfo": "QuotaState: BLOCKED" 783 } 784 ], 785 "code": 403, 786 "message": "Access Not Configured" 787 } 788} 789 790--batch_foobarbaz--""" 791 792 793BATCH_RESPONSE_WITH_401 = b"""--batch_foobarbaz 794Content-Type: application/http 795Content-Transfer-Encoding: binary 796Content-ID: <randomness + 1> 797 798HTTP/1.1 401 Authorization Required 799Content-Type: application/json 800Content-Length: 14 801ETag: "etag/pony"\r\n\r\n{"error": {"message": 802 "Authorizaton failed."}} 803 804--batch_foobarbaz 805Content-Type: application/http 806Content-Transfer-Encoding: binary 807Content-ID: <randomness + 2> 808 809HTTP/1.1 200 OK 810Content-Type: application/json 811Content-Length: 14 812ETag: "etag/sheep"\r\n\r\n{"baz": "qux"} 813--batch_foobarbaz--""" 814 815 816BATCH_SINGLE_RESPONSE = b"""--batch_foobarbaz 817Content-Type: application/http 818Content-Transfer-Encoding: binary 819Content-ID: <randomness + 1> 820 821HTTP/1.1 200 OK 822Content-Type: application/json 823Content-Length: 14 824ETag: "etag/pony"\r\n\r\n{"foo": 42} 825--batch_foobarbaz--""" 826 827 828USER_RATE_LIMIT_EXCEEDED_RESPONSE_NO_STATUS = """{ 829 "error": { 830 "errors": [ 831 { 832 "domain": "usageLimits", 833 "reason": "userRateLimitExceeded", 834 "message": "User Rate Limit Exceeded" 835 } 836 ], 837 "code": 403, 838 "message": "User Rate Limit Exceeded" 839 } 840}""" 841 842USER_RATE_LIMIT_EXCEEDED_RESPONSE_WITH_STATUS = """{ 843 "error": { 844 "errors": [ 845 { 846 "domain": "usageLimits", 847 "reason": "userRateLimitExceeded", 848 "message": "User Rate Limit Exceeded" 849 } 850 ], 851 "code": 403, 852 "message": "User Rate Limit Exceeded", 853 "status": "PERMISSION_DENIED" 854 } 855}""" 856 857RATE_LIMIT_EXCEEDED_RESPONSE = """{ 858 "error": { 859 "errors": [ 860 { 861 "domain": "usageLimits", 862 "reason": "rateLimitExceeded", 863 "message": "Rate Limit Exceeded" 864 } 865 ], 866 "code": 403, 867 "message": "Rate Limit Exceeded" 868 } 869}""" 870 871 872NOT_CONFIGURED_RESPONSE = """{ 873 "error": { 874 "errors": [ 875 { 876 "domain": "usageLimits", 877 "reason": "accessNotConfigured", 878 "message": "Access Not Configured" 879 } 880 ], 881 "code": 403, 882 "message": "Access Not Configured" 883 } 884}""" 885 886LIST_NOT_CONFIGURED_RESPONSE = """[ 887 "error": { 888 "errors": [ 889 { 890 "domain": "usageLimits", 891 "reason": "accessNotConfigured", 892 "message": "Access Not Configured" 893 } 894 ], 895 "code": 403, 896 "message": "Access Not Configured" 897 } 898]""" 899 900 901class Callbacks(object): 902 def __init__(self): 903 self.responses = {} 904 self.exceptions = {} 905 906 def f(self, request_id, response, exception): 907 self.responses[request_id] = response 908 self.exceptions[request_id] = exception 909 910 911class TestHttpRequest(unittest.TestCase): 912 def test_unicode(self): 913 http = HttpMock(datafile("zoo.json"), headers={"status": "200"}) 914 model = JsonModel() 915 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar" 916 method = u"POST" 917 request = HttpRequest( 918 http, 919 model.response, 920 uri, 921 method=method, 922 body=u"{}", 923 headers={"content-type": "application/json"}, 924 ) 925 request.execute() 926 self.assertEqual(uri, http.uri) 927 self.assertEqual(str, type(http.uri)) 928 self.assertEqual(method, http.method) 929 self.assertEqual(str, type(http.method)) 930 931 def test_empty_content_type(self): 932 """Test for #284""" 933 http = HttpMock(None, headers={"status": 200}) 934 uri = u"https://www.googleapis.com/someapi/v1/upload/?foo=bar" 935 method = u"POST" 936 request = HttpRequest( 937 http, _postproc_none, uri, method=method, headers={"content-type": ""} 938 ) 939 request.execute() 940 self.assertEqual("", http.headers.get("content-type")) 941 942 def test_no_retry_connection_errors(self): 943 model = JsonModel() 944 request = HttpRequest( 945 HttpMockWithNonRetriableErrors(1, {"status": "200"}, '{"foo": "bar"}'), 946 model.response, 947 u"https://www.example.com/json_api_endpoint", 948 ) 949 request._sleep = lambda _x: 0 # do nothing 950 request._rand = lambda: 10 951 with self.assertRaises(OSError): 952 response = request.execute(num_retries=3) 953 954 def test_retry_connection_errors_non_resumable(self): 955 model = JsonModel() 956 request = HttpRequest( 957 HttpMockWithErrors(5, {"status": "200"}, '{"foo": "bar"}'), 958 model.response, 959 u"https://www.example.com/json_api_endpoint", 960 ) 961 request._sleep = lambda _x: 0 # do nothing 962 request._rand = lambda: 10 963 response = request.execute(num_retries=5) 964 self.assertEqual({u"foo": u"bar"}, response) 965 966 def test_retry_connection_errors_resumable(self): 967 with open(datafile("small.png"), "rb") as small_png_file: 968 small_png_fd = io.BytesIO(small_png_file.read()) 969 upload = MediaIoBaseUpload( 970 fd=small_png_fd, mimetype="image/png", chunksize=500, resumable=True 971 ) 972 model = JsonModel() 973 974 request = HttpRequest( 975 HttpMockWithErrors( 976 5, {"status": "200", "location": "location"}, '{"foo": "bar"}' 977 ), 978 model.response, 979 u"https://www.example.com/file_upload", 980 method="POST", 981 resumable=upload, 982 ) 983 request._sleep = lambda _x: 0 # do nothing 984 request._rand = lambda: 10 985 response = request.execute(num_retries=5) 986 self.assertEqual({u"foo": u"bar"}, response) 987 988 def test_retry(self): 989 num_retries = 6 990 resp_seq = [({"status": "500"}, "")] * (num_retries - 4) 991 resp_seq.append(({"status": "403"}, RATE_LIMIT_EXCEEDED_RESPONSE)) 992 resp_seq.append(({"status": "403"}, USER_RATE_LIMIT_EXCEEDED_RESPONSE_NO_STATUS)) 993 resp_seq.append(({"status": "403"}, USER_RATE_LIMIT_EXCEEDED_RESPONSE_WITH_STATUS)) 994 resp_seq.append(({"status": "429"}, "")) 995 resp_seq.append(({"status": "200"}, "{}")) 996 997 http = HttpMockSequence(resp_seq) 998 model = JsonModel() 999 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar" 1000 method = u"POST" 1001 request = HttpRequest( 1002 http, 1003 model.response, 1004 uri, 1005 method=method, 1006 body=u"{}", 1007 headers={"content-type": "application/json"}, 1008 ) 1009 1010 sleeptimes = [] 1011 request._sleep = lambda x: sleeptimes.append(x) 1012 request._rand = lambda: 10 1013 1014 request.execute(num_retries=num_retries) 1015 1016 self.assertEqual(num_retries, len(sleeptimes)) 1017 for retry_num in range(num_retries): 1018 self.assertEqual(10 * 2 ** (retry_num + 1), sleeptimes[retry_num]) 1019 1020 def test_no_retry_succeeds(self): 1021 num_retries = 5 1022 resp_seq = [({"status": "200"}, "{}")] * (num_retries) 1023 1024 http = HttpMockSequence(resp_seq) 1025 model = JsonModel() 1026 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar" 1027 method = u"POST" 1028 request = HttpRequest( 1029 http, 1030 model.response, 1031 uri, 1032 method=method, 1033 body=u"{}", 1034 headers={"content-type": "application/json"}, 1035 ) 1036 1037 sleeptimes = [] 1038 request._sleep = lambda x: sleeptimes.append(x) 1039 request._rand = lambda: 10 1040 1041 request.execute(num_retries=num_retries) 1042 1043 self.assertEqual(0, len(sleeptimes)) 1044 1045 def test_no_retry_fails_fast(self): 1046 http = HttpMockSequence([({"status": "500"}, ""), ({"status": "200"}, "{}")]) 1047 model = JsonModel() 1048 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar" 1049 method = u"POST" 1050 request = HttpRequest( 1051 http, 1052 model.response, 1053 uri, 1054 method=method, 1055 body=u"{}", 1056 headers={"content-type": "application/json"}, 1057 ) 1058 1059 request._rand = lambda: 1.0 1060 request._sleep = mock.MagicMock() 1061 1062 with self.assertRaises(HttpError): 1063 request.execute() 1064 request._sleep.assert_not_called() 1065 1066 def test_no_retry_403_not_configured_fails_fast(self): 1067 http = HttpMockSequence( 1068 [({"status": "403"}, NOT_CONFIGURED_RESPONSE), ({"status": "200"}, "{}")] 1069 ) 1070 model = JsonModel() 1071 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar" 1072 method = u"POST" 1073 request = HttpRequest( 1074 http, 1075 model.response, 1076 uri, 1077 method=method, 1078 body=u"{}", 1079 headers={"content-type": "application/json"}, 1080 ) 1081 1082 request._rand = lambda: 1.0 1083 request._sleep = mock.MagicMock() 1084 1085 with self.assertRaises(HttpError): 1086 request.execute() 1087 request._sleep.assert_not_called() 1088 1089 def test_no_retry_403_fails_fast(self): 1090 http = HttpMockSequence([({"status": "403"}, ""), ({"status": "200"}, "{}")]) 1091 model = JsonModel() 1092 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar" 1093 method = u"POST" 1094 request = HttpRequest( 1095 http, 1096 model.response, 1097 uri, 1098 method=method, 1099 body=u"{}", 1100 headers={"content-type": "application/json"}, 1101 ) 1102 1103 request._rand = lambda: 1.0 1104 request._sleep = mock.MagicMock() 1105 1106 with self.assertRaises(HttpError): 1107 request.execute() 1108 request._sleep.assert_not_called() 1109 1110 def test_no_retry_401_fails_fast(self): 1111 http = HttpMockSequence([({"status": "401"}, ""), ({"status": "200"}, "{}")]) 1112 model = JsonModel() 1113 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar" 1114 method = u"POST" 1115 request = HttpRequest( 1116 http, 1117 model.response, 1118 uri, 1119 method=method, 1120 body=u"{}", 1121 headers={"content-type": "application/json"}, 1122 ) 1123 1124 request._rand = lambda: 1.0 1125 request._sleep = mock.MagicMock() 1126 1127 with self.assertRaises(HttpError): 1128 request.execute() 1129 request._sleep.assert_not_called() 1130 1131 def test_no_retry_403_list_fails(self): 1132 http = HttpMockSequence( 1133 [ 1134 ({"status": "403"}, LIST_NOT_CONFIGURED_RESPONSE), 1135 ({"status": "200"}, "{}"), 1136 ] 1137 ) 1138 model = JsonModel() 1139 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar" 1140 method = u"POST" 1141 request = HttpRequest( 1142 http, 1143 model.response, 1144 uri, 1145 method=method, 1146 body=u"{}", 1147 headers={"content-type": "application/json"}, 1148 ) 1149 1150 request._rand = lambda: 1.0 1151 request._sleep = mock.MagicMock() 1152 1153 with self.assertRaises(HttpError): 1154 request.execute() 1155 request._sleep.assert_not_called() 1156 1157 def test_null_postproc(self): 1158 resp, content = HttpRequest.null_postproc("foo", "bar") 1159 self.assertEqual(resp, "foo") 1160 self.assertEqual(content, "bar") 1161 1162 1163class TestBatch(unittest.TestCase): 1164 def setUp(self): 1165 model = JsonModel() 1166 self.request1 = HttpRequest( 1167 None, 1168 model.response, 1169 "https://www.googleapis.com/someapi/v1/collection/?foo=bar", 1170 method="POST", 1171 body="{}", 1172 headers={"content-type": "application/json"}, 1173 ) 1174 1175 self.request2 = HttpRequest( 1176 None, 1177 model.response, 1178 "https://www.googleapis.com/someapi/v1/collection/?foo=bar", 1179 method="GET", 1180 body="", 1181 headers={"content-type": "application/json"}, 1182 ) 1183 1184 def test_id_to_from_content_id_header(self): 1185 batch = BatchHttpRequest() 1186 self.assertEqual("12", batch._header_to_id(batch._id_to_header("12"))) 1187 1188 def test_invalid_content_id_header(self): 1189 batch = BatchHttpRequest() 1190 self.assertRaises(BatchError, batch._header_to_id, "[foo+x]") 1191 self.assertRaises(BatchError, batch._header_to_id, "foo+1") 1192 self.assertRaises(BatchError, batch._header_to_id, "<foo>") 1193 1194 def test_serialize_request(self): 1195 batch = BatchHttpRequest() 1196 request = HttpRequest( 1197 None, 1198 None, 1199 "https://www.googleapis.com/someapi/v1/collection/?foo=bar", 1200 method="POST", 1201 body=u"{}", 1202 headers={"content-type": "application/json"}, 1203 methodId=None, 1204 resumable=None, 1205 ) 1206 s = batch._serialize_request(request).splitlines() 1207 self.assertEqual(EXPECTED.splitlines(), s) 1208 1209 def test_serialize_request_media_body(self): 1210 batch = BatchHttpRequest() 1211 f = open(datafile("small.png"), "rb") 1212 body = f.read() 1213 f.close() 1214 1215 request = HttpRequest( 1216 None, 1217 None, 1218 "https://www.googleapis.com/someapi/v1/collection/?foo=bar", 1219 method="POST", 1220 body=body, 1221 headers={"content-type": "application/json"}, 1222 methodId=None, 1223 resumable=None, 1224 ) 1225 # Just testing it shouldn't raise an exception. 1226 s = batch._serialize_request(request).splitlines() 1227 1228 def test_serialize_request_no_body(self): 1229 batch = BatchHttpRequest() 1230 request = HttpRequest( 1231 None, 1232 None, 1233 "https://www.googleapis.com/someapi/v1/collection/?foo=bar", 1234 method="POST", 1235 body=b"", 1236 headers={"content-type": "application/json"}, 1237 methodId=None, 1238 resumable=None, 1239 ) 1240 s = batch._serialize_request(request).splitlines() 1241 self.assertEqual(NO_BODY_EXPECTED.splitlines(), s) 1242 1243 def test_serialize_get_request_no_body(self): 1244 batch = BatchHttpRequest() 1245 request = HttpRequest( 1246 None, 1247 None, 1248 "https://www.googleapis.com/someapi/v1/collection/?foo=bar", 1249 method="GET", 1250 body=None, 1251 headers={"content-type": "application/json"}, 1252 methodId=None, 1253 resumable=None, 1254 ) 1255 s = batch._serialize_request(request).splitlines() 1256 self.assertEqual(NO_BODY_EXPECTED_GET.splitlines(), s) 1257 1258 def test_deserialize_response(self): 1259 batch = BatchHttpRequest() 1260 resp, content = batch._deserialize_response(RESPONSE) 1261 1262 self.assertEqual(200, resp.status) 1263 self.assertEqual("OK", resp.reason) 1264 self.assertEqual(11, resp.version) 1265 self.assertEqual('{"answer": 42}', content) 1266 1267 def test_new_id(self): 1268 batch = BatchHttpRequest() 1269 1270 id_ = batch._new_id() 1271 self.assertEqual("1", id_) 1272 1273 id_ = batch._new_id() 1274 self.assertEqual("2", id_) 1275 1276 batch.add(self.request1, request_id="3") 1277 1278 id_ = batch._new_id() 1279 self.assertEqual("4", id_) 1280 1281 def test_add(self): 1282 batch = BatchHttpRequest() 1283 batch.add(self.request1, request_id="1") 1284 self.assertRaises(KeyError, batch.add, self.request1, request_id="1") 1285 1286 def test_add_fail_for_over_limit(self): 1287 from googleapiclient.http import MAX_BATCH_LIMIT 1288 1289 batch = BatchHttpRequest() 1290 for i in range(0, MAX_BATCH_LIMIT): 1291 batch.add( 1292 HttpRequest( 1293 None, 1294 None, 1295 "https://www.googleapis.com/someapi/v1/collection/?foo=bar", 1296 method="POST", 1297 body="{}", 1298 headers={"content-type": "application/json"}, 1299 ) 1300 ) 1301 self.assertRaises(BatchError, batch.add, self.request1) 1302 1303 def test_add_fail_for_resumable(self): 1304 batch = BatchHttpRequest() 1305 1306 upload = MediaFileUpload(datafile("small.png"), chunksize=500, resumable=True) 1307 self.request1.resumable = upload 1308 with self.assertRaises(BatchError) as batch_error: 1309 batch.add(self.request1, request_id="1") 1310 str(batch_error.exception) 1311 1312 def test_execute_empty_batch_no_http(self): 1313 batch = BatchHttpRequest() 1314 ret = batch.execute() 1315 self.assertEqual(None, ret) 1316 1317 def test_execute(self): 1318 batch = BatchHttpRequest() 1319 callbacks = Callbacks() 1320 1321 batch.add(self.request1, callback=callbacks.f) 1322 batch.add(self.request2, callback=callbacks.f) 1323 http = HttpMockSequence( 1324 [ 1325 ( 1326 { 1327 "status": "200", 1328 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1329 }, 1330 BATCH_RESPONSE, 1331 ) 1332 ] 1333 ) 1334 batch.execute(http=http) 1335 self.assertEqual({"foo": 42}, callbacks.responses["1"]) 1336 self.assertEqual(None, callbacks.exceptions["1"]) 1337 self.assertEqual({"baz": "qux"}, callbacks.responses["2"]) 1338 self.assertEqual(None, callbacks.exceptions["2"]) 1339 1340 def test_execute_request_body(self): 1341 batch = BatchHttpRequest() 1342 1343 batch.add(self.request1) 1344 batch.add(self.request2) 1345 http = HttpMockSequence( 1346 [ 1347 ( 1348 { 1349 "status": "200", 1350 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1351 }, 1352 "echo_request_body", 1353 ) 1354 ] 1355 ) 1356 try: 1357 batch.execute(http=http) 1358 self.fail("Should raise exception") 1359 except BatchError as e: 1360 boundary, _ = e.content.split(None, 1) 1361 self.assertEqual("--", boundary[:2]) 1362 parts = e.content.split(boundary) 1363 self.assertEqual(4, len(parts)) 1364 self.assertEqual("", parts[0]) 1365 self.assertEqual("--", parts[3].rstrip()) 1366 header = parts[1].splitlines()[1] 1367 self.assertEqual("Content-Type: application/http", header) 1368 1369 def test_execute_request_body_with_custom_long_request_ids(self): 1370 batch = BatchHttpRequest() 1371 1372 batch.add(self.request1, request_id="abc" * 20) 1373 batch.add(self.request2, request_id="def" * 20) 1374 http = HttpMockSequence( 1375 [ 1376 ( 1377 { 1378 "status": "200", 1379 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1380 }, 1381 "echo_request_body", 1382 ) 1383 ] 1384 ) 1385 try: 1386 batch.execute(http=http) 1387 self.fail("Should raise exception") 1388 except BatchError as e: 1389 boundary, _ = e.content.split(None, 1) 1390 self.assertEqual("--", boundary[:2]) 1391 parts = e.content.split(boundary) 1392 self.assertEqual(4, len(parts)) 1393 self.assertEqual("", parts[0]) 1394 self.assertEqual("--", parts[3].rstrip()) 1395 for partindex, request_id in ((1, "abc" * 20), (2, "def" * 20)): 1396 lines = parts[partindex].splitlines() 1397 for n, line in enumerate(lines): 1398 if line.startswith("Content-ID:"): 1399 # assert correct header folding 1400 self.assertTrue(line.endswith("+"), line) 1401 header_continuation = lines[n + 1] 1402 self.assertEqual( 1403 header_continuation, 1404 " %s>" % request_id, 1405 header_continuation, 1406 ) 1407 1408 def test_execute_initial_refresh_oauth2(self): 1409 batch = BatchHttpRequest() 1410 callbacks = Callbacks() 1411 cred = MockCredentials("Foo", expired=True) 1412 1413 http = HttpMockSequence( 1414 [ 1415 ( 1416 { 1417 "status": "200", 1418 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1419 }, 1420 BATCH_SINGLE_RESPONSE, 1421 ) 1422 ] 1423 ) 1424 1425 cred.authorize(http) 1426 1427 batch.add(self.request1, callback=callbacks.f) 1428 batch.execute(http=http) 1429 1430 self.assertEqual({"foo": 42}, callbacks.responses["1"]) 1431 self.assertIsNone(callbacks.exceptions["1"]) 1432 1433 self.assertEqual(1, cred._refreshed) 1434 1435 self.assertEqual(1, cred._authorized) 1436 1437 self.assertEqual(1, cred._applied) 1438 1439 def test_execute_refresh_and_retry_on_401(self): 1440 batch = BatchHttpRequest() 1441 callbacks = Callbacks() 1442 cred_1 = MockCredentials("Foo") 1443 cred_2 = MockCredentials("Bar") 1444 1445 http = HttpMockSequence( 1446 [ 1447 ( 1448 { 1449 "status": "200", 1450 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1451 }, 1452 BATCH_RESPONSE_WITH_401, 1453 ), 1454 ( 1455 { 1456 "status": "200", 1457 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1458 }, 1459 BATCH_SINGLE_RESPONSE, 1460 ), 1461 ] 1462 ) 1463 1464 creds_http_1 = HttpMockSequence([]) 1465 cred_1.authorize(creds_http_1) 1466 1467 creds_http_2 = HttpMockSequence([]) 1468 cred_2.authorize(creds_http_2) 1469 1470 self.request1.http = creds_http_1 1471 self.request2.http = creds_http_2 1472 1473 batch.add(self.request1, callback=callbacks.f) 1474 batch.add(self.request2, callback=callbacks.f) 1475 batch.execute(http=http) 1476 1477 self.assertEqual({"foo": 42}, callbacks.responses["1"]) 1478 self.assertEqual(None, callbacks.exceptions["1"]) 1479 self.assertEqual({"baz": "qux"}, callbacks.responses["2"]) 1480 self.assertEqual(None, callbacks.exceptions["2"]) 1481 1482 self.assertEqual(1, cred_1._refreshed) 1483 self.assertEqual(0, cred_2._refreshed) 1484 1485 self.assertEqual(1, cred_1._authorized) 1486 self.assertEqual(1, cred_2._authorized) 1487 1488 self.assertEqual(1, cred_2._applied) 1489 self.assertEqual(2, cred_1._applied) 1490 1491 def test_http_errors_passed_to_callback(self): 1492 batch = BatchHttpRequest() 1493 callbacks = Callbacks() 1494 cred_1 = MockCredentials("Foo") 1495 cred_2 = MockCredentials("Bar") 1496 1497 http = HttpMockSequence( 1498 [ 1499 ( 1500 { 1501 "status": "200", 1502 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1503 }, 1504 BATCH_RESPONSE_WITH_401, 1505 ), 1506 ( 1507 { 1508 "status": "200", 1509 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1510 }, 1511 BATCH_RESPONSE_WITH_401, 1512 ), 1513 ] 1514 ) 1515 1516 creds_http_1 = HttpMockSequence([]) 1517 cred_1.authorize(creds_http_1) 1518 1519 creds_http_2 = HttpMockSequence([]) 1520 cred_2.authorize(creds_http_2) 1521 1522 self.request1.http = creds_http_1 1523 self.request2.http = creds_http_2 1524 1525 batch.add(self.request1, callback=callbacks.f) 1526 batch.add(self.request2, callback=callbacks.f) 1527 batch.execute(http=http) 1528 1529 self.assertEqual(None, callbacks.responses["1"]) 1530 self.assertEqual(401, callbacks.exceptions["1"].resp.status) 1531 self.assertEqual( 1532 "Authorization Required", callbacks.exceptions["1"].resp.reason 1533 ) 1534 self.assertEqual({u"baz": u"qux"}, callbacks.responses["2"]) 1535 self.assertEqual(None, callbacks.exceptions["2"]) 1536 1537 def test_execute_global_callback(self): 1538 callbacks = Callbacks() 1539 batch = BatchHttpRequest(callback=callbacks.f) 1540 1541 batch.add(self.request1) 1542 batch.add(self.request2) 1543 http = HttpMockSequence( 1544 [ 1545 ( 1546 { 1547 "status": "200", 1548 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1549 }, 1550 BATCH_RESPONSE, 1551 ) 1552 ] 1553 ) 1554 batch.execute(http=http) 1555 self.assertEqual({"foo": 42}, callbacks.responses["1"]) 1556 self.assertEqual({"baz": "qux"}, callbacks.responses["2"]) 1557 1558 def test_execute_batch_http_error(self): 1559 callbacks = Callbacks() 1560 batch = BatchHttpRequest(callback=callbacks.f) 1561 1562 batch.add(self.request1) 1563 batch.add(self.request2) 1564 http = HttpMockSequence( 1565 [ 1566 ( 1567 { 1568 "status": "200", 1569 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"', 1570 }, 1571 BATCH_ERROR_RESPONSE, 1572 ) 1573 ] 1574 ) 1575 batch.execute(http=http) 1576 self.assertEqual({"foo": 42}, callbacks.responses["1"]) 1577 expected = ( 1578 "<HttpError 403 when requesting " 1579 "https://www.googleapis.com/someapi/v1/collection/?foo=bar returned " 1580 '"Access Not Configured". ' 1581 "Details: \"[{'domain': 'usageLimits', 'reason': 'accessNotConfigured', 'message': 'Access Not Configured', 'debugInfo': 'QuotaState: BLOCKED'}]\">" 1582 ) 1583 self.assertEqual(expected, str(callbacks.exceptions["2"])) 1584 1585 1586class TestRequestUriTooLong(unittest.TestCase): 1587 def test_turn_get_into_post(self): 1588 def _postproc(resp, content): 1589 return content 1590 1591 http = HttpMockSequence( 1592 [ 1593 ({"status": "200"}, "echo_request_body"), 1594 ({"status": "200"}, "echo_request_headers"), 1595 ] 1596 ) 1597 1598 # Send a long query parameter. 1599 query = {"q": "a" * MAX_URI_LENGTH + "?&"} 1600 req = HttpRequest( 1601 http, 1602 _postproc, 1603 "http://example.com?" + urllib.parse.urlencode(query), 1604 method="GET", 1605 body=None, 1606 headers={}, 1607 methodId="foo", 1608 resumable=None, 1609 ) 1610 1611 # Query parameters should be sent in the body. 1612 response = req.execute() 1613 self.assertEqual(b"q=" + b"a" * MAX_URI_LENGTH + b"%3F%26", response) 1614 1615 # Extra headers should be set. 1616 response = req.execute() 1617 self.assertEqual("GET", response["x-http-method-override"]) 1618 self.assertEqual(str(MAX_URI_LENGTH + 8), response["content-length"]) 1619 self.assertEqual("application/x-www-form-urlencoded", response["content-type"]) 1620 1621 1622class TestStreamSlice(unittest.TestCase): 1623 """Test _StreamSlice.""" 1624 1625 def setUp(self): 1626 self.stream = io.BytesIO(b"0123456789") 1627 1628 def test_read(self): 1629 s = _StreamSlice(self.stream, 0, 4) 1630 self.assertEqual(b"", s.read(0)) 1631 self.assertEqual(b"0", s.read(1)) 1632 self.assertEqual(b"123", s.read()) 1633 1634 def test_read_too_much(self): 1635 s = _StreamSlice(self.stream, 1, 4) 1636 self.assertEqual(b"1234", s.read(6)) 1637 1638 def test_read_all(self): 1639 s = _StreamSlice(self.stream, 2, 1) 1640 self.assertEqual(b"2", s.read(-1)) 1641 1642 1643class TestResponseCallback(unittest.TestCase): 1644 """Test adding callbacks to responses.""" 1645 1646 def test_ensure_response_callback(self): 1647 m = JsonModel() 1648 request = HttpRequest( 1649 None, 1650 m.response, 1651 "https://www.googleapis.com/someapi/v1/collection/?foo=bar", 1652 method="POST", 1653 body="{}", 1654 headers={"content-type": "application/json"}, 1655 ) 1656 h = HttpMockSequence([({"status": 200}, "{}")]) 1657 responses = [] 1658 1659 def _on_response(resp, responses=responses): 1660 responses.append(resp) 1661 1662 request.add_response_callback(_on_response) 1663 request.execute(http=h) 1664 self.assertEqual(1, len(responses)) 1665 1666 1667class TestHttpMock(unittest.TestCase): 1668 def test_default_response_headers(self): 1669 http = HttpMock(datafile("zoo.json")) 1670 resp, content = http.request("http://example.com") 1671 self.assertEqual(resp.status, 200) 1672 1673 def test_error_response(self): 1674 http = HttpMock(datafile("bad_request.json"), {"status": "400"}) 1675 model = JsonModel() 1676 request = HttpRequest( 1677 http, 1678 model.response, 1679 "https://www.googleapis.com/someapi/v1/collection/?foo=bar", 1680 method="GET", 1681 headers={}, 1682 ) 1683 self.assertRaises(HttpError, request.execute) 1684 1685 1686class TestHttpBuild(unittest.TestCase): 1687 original_socket_default_timeout = None 1688 1689 @classmethod 1690 def setUpClass(cls): 1691 cls.original_socket_default_timeout = socket.getdefaulttimeout() 1692 1693 @classmethod 1694 def tearDownClass(cls): 1695 socket.setdefaulttimeout(cls.original_socket_default_timeout) 1696 1697 def test_build_http_sets_default_timeout_if_none_specified(self): 1698 socket.setdefaulttimeout(None) 1699 http = build_http() 1700 self.assertIsInstance(http.timeout, int) 1701 self.assertGreater(http.timeout, 0) 1702 1703 def test_build_http_default_timeout_can_be_overridden(self): 1704 socket.setdefaulttimeout(1.5) 1705 http = build_http() 1706 self.assertAlmostEqual(http.timeout, 1.5, delta=0.001) 1707 1708 def test_build_http_default_timeout_can_be_set_to_zero(self): 1709 socket.setdefaulttimeout(0) 1710 http = build_http() 1711 self.assertEqual(http.timeout, 0) 1712 1713 def test_build_http_default_308_is_excluded_as_redirect(self): 1714 http = build_http() 1715 self.assertTrue(308 not in http.redirect_codes) 1716 1717 1718if __name__ == "__main__": 1719 logging.getLogger().setLevel(logging.ERROR) 1720 unittest.main() 1721