1#!/usr/bin env python 2 3from tests.compat import mock, unittest 4from httpretty import HTTPretty 5 6import json 7import requests 8 9from boto.cloudsearch.search import SearchConnection, SearchServiceException 10from boto.compat import six, map 11 12HOSTNAME = "search-demo-userdomain.us-east-1.cloudsearch.amazonaws.com" 13FULL_URL = 'http://%s/2011-02-01/search' % HOSTNAME 14 15 16class CloudSearchSearchBaseTest(unittest.TestCase): 17 18 hits = [ 19 { 20 'id': '12341', 21 'title': 'Document 1', 22 }, 23 { 24 'id': '12342', 25 'title': 'Document 2', 26 }, 27 { 28 'id': '12343', 29 'title': 'Document 3', 30 }, 31 { 32 'id': '12344', 33 'title': 'Document 4', 34 }, 35 { 36 'id': '12345', 37 'title': 'Document 5', 38 }, 39 { 40 'id': '12346', 41 'title': 'Document 6', 42 }, 43 { 44 'id': '12347', 45 'title': 'Document 7', 46 }, 47 ] 48 49 content_type = "text/xml" 50 response_status = 200 51 52 def get_args(self, requestline): 53 (_, request, _) = requestline.split(b" ") 54 (_, request) = request.split(b"?", 1) 55 args = six.moves.urllib.parse.parse_qs(request) 56 return args 57 58 def setUp(self): 59 HTTPretty.enable() 60 body = self.response 61 62 if not isinstance(body, bytes): 63 body = json.dumps(body).encode('utf-8') 64 65 HTTPretty.register_uri(HTTPretty.GET, FULL_URL, 66 body=body, 67 content_type=self.content_type, 68 status=self.response_status) 69 70 def tearDown(self): 71 HTTPretty.disable() 72 73class CloudSearchSearchTest(CloudSearchSearchBaseTest): 74 response = { 75 'rank': '-text_relevance', 76 'match-expr': "Test", 77 'hits': { 78 'found': 30, 79 'start': 0, 80 'hit': CloudSearchSearchBaseTest.hits 81 }, 82 'info': { 83 'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08', 84 'time-ms': 2, 85 'cpu-time-ms': 0 86 } 87 88 } 89 90 def test_cloudsearch_qsearch(self): 91 search = SearchConnection(endpoint=HOSTNAME) 92 93 search.search(q='Test') 94 95 args = self.get_args(HTTPretty.last_request.raw_requestline) 96 97 self.assertEqual(args[b'q'], [b"Test"]) 98 self.assertEqual(args[b'start'], [b"0"]) 99 self.assertEqual(args[b'size'], [b"10"]) 100 101 def test_cloudsearch_bqsearch(self): 102 search = SearchConnection(endpoint=HOSTNAME) 103 104 search.search(bq="'Test'") 105 106 args = self.get_args(HTTPretty.last_request.raw_requestline) 107 108 self.assertEqual(args[b'bq'], [b"'Test'"]) 109 110 def test_cloudsearch_search_details(self): 111 search = SearchConnection(endpoint=HOSTNAME) 112 113 search.search(q='Test', size=50, start=20) 114 115 args = self.get_args(HTTPretty.last_request.raw_requestline) 116 117 self.assertEqual(args[b'q'], [b"Test"]) 118 self.assertEqual(args[b'size'], [b"50"]) 119 self.assertEqual(args[b'start'], [b"20"]) 120 121 def test_cloudsearch_facet_single(self): 122 search = SearchConnection(endpoint=HOSTNAME) 123 124 search.search(q='Test', facet=["Author"]) 125 126 args = self.get_args(HTTPretty.last_request.raw_requestline) 127 128 self.assertEqual(args[b'facet'], [b"Author"]) 129 130 def test_cloudsearch_facet_multiple(self): 131 search = SearchConnection(endpoint=HOSTNAME) 132 133 search.search(q='Test', facet=["author", "cat"]) 134 135 args = self.get_args(HTTPretty.last_request.raw_requestline) 136 137 self.assertEqual(args[b'facet'], [b"author,cat"]) 138 139 def test_cloudsearch_facet_constraint_single(self): 140 search = SearchConnection(endpoint=HOSTNAME) 141 142 search.search( 143 q='Test', 144 facet_constraints={'author': "'John Smith','Mark Smith'"}) 145 146 args = self.get_args(HTTPretty.last_request.raw_requestline) 147 148 self.assertEqual(args[b'facet-author-constraints'], 149 [b"'John Smith','Mark Smith'"]) 150 151 def test_cloudsearch_facet_constraint_multiple(self): 152 search = SearchConnection(endpoint=HOSTNAME) 153 154 search.search( 155 q='Test', 156 facet_constraints={'author': "'John Smith','Mark Smith'", 157 'category': "'News','Reviews'"}) 158 159 args = self.get_args(HTTPretty.last_request.raw_requestline) 160 161 self.assertEqual(args[b'facet-author-constraints'], 162 [b"'John Smith','Mark Smith'"]) 163 self.assertEqual(args[b'facet-category-constraints'], 164 [b"'News','Reviews'"]) 165 166 def test_cloudsearch_facet_sort_single(self): 167 search = SearchConnection(endpoint=HOSTNAME) 168 169 search.search(q='Test', facet_sort={'author': 'alpha'}) 170 171 args = self.get_args(HTTPretty.last_request.raw_requestline) 172 173 self.assertEqual(args[b'facet-author-sort'], [b'alpha']) 174 175 def test_cloudsearch_facet_sort_multiple(self): 176 search = SearchConnection(endpoint=HOSTNAME) 177 178 search.search(q='Test', facet_sort={'author': 'alpha', 179 'cat': 'count'}) 180 181 args = self.get_args(HTTPretty.last_request.raw_requestline) 182 183 self.assertEqual(args[b'facet-author-sort'], [b'alpha']) 184 self.assertEqual(args[b'facet-cat-sort'], [b'count']) 185 186 def test_cloudsearch_top_n_single(self): 187 search = SearchConnection(endpoint=HOSTNAME) 188 189 search.search(q='Test', facet_top_n={'author': 5}) 190 191 args = self.get_args(HTTPretty.last_request.raw_requestline) 192 193 self.assertEqual(args[b'facet-author-top-n'], [b'5']) 194 195 def test_cloudsearch_top_n_multiple(self): 196 search = SearchConnection(endpoint=HOSTNAME) 197 198 search.search(q='Test', facet_top_n={'author': 5, 'cat': 10}) 199 200 args = self.get_args(HTTPretty.last_request.raw_requestline) 201 202 self.assertEqual(args[b'facet-author-top-n'], [b'5']) 203 self.assertEqual(args[b'facet-cat-top-n'], [b'10']) 204 205 def test_cloudsearch_rank_single(self): 206 search = SearchConnection(endpoint=HOSTNAME) 207 208 search.search(q='Test', rank=["date"]) 209 210 args = self.get_args(HTTPretty.last_request.raw_requestline) 211 212 self.assertEqual(args[b'rank'], [b'date']) 213 214 def test_cloudsearch_rank_multiple(self): 215 search = SearchConnection(endpoint=HOSTNAME) 216 217 search.search(q='Test', rank=["date", "score"]) 218 219 args = self.get_args(HTTPretty.last_request.raw_requestline) 220 221 self.assertEqual(args[b'rank'], [b'date,score']) 222 223 def test_cloudsearch_result_fields_single(self): 224 search = SearchConnection(endpoint=HOSTNAME) 225 226 search.search(q='Test', return_fields=['author']) 227 228 args = self.get_args(HTTPretty.last_request.raw_requestline) 229 230 self.assertEqual(args[b'return-fields'], [b'author']) 231 232 def test_cloudsearch_result_fields_multiple(self): 233 search = SearchConnection(endpoint=HOSTNAME) 234 235 search.search(q='Test', return_fields=['author', 'title']) 236 237 args = self.get_args(HTTPretty.last_request.raw_requestline) 238 239 self.assertEqual(args[b'return-fields'], [b'author,title']) 240 241 def test_cloudsearch_t_field_single(self): 242 search = SearchConnection(endpoint=HOSTNAME) 243 244 search.search(q='Test', t={'year': '2001..2007'}) 245 246 args = self.get_args(HTTPretty.last_request.raw_requestline) 247 248 self.assertEqual(args[b't-year'], [b'2001..2007']) 249 250 def test_cloudsearch_t_field_multiple(self): 251 search = SearchConnection(endpoint=HOSTNAME) 252 253 search.search(q='Test', t={'year': '2001..2007', 'score': '10..50'}) 254 255 args = self.get_args(HTTPretty.last_request.raw_requestline) 256 257 self.assertEqual(args[b't-year'], [b'2001..2007']) 258 self.assertEqual(args[b't-score'], [b'10..50']) 259 260 def test_cloudsearch_results_meta(self): 261 """Check returned metadata is parsed correctly""" 262 search = SearchConnection(endpoint=HOSTNAME) 263 264 results = search.search(q='Test') 265 266 # These rely on the default response which is fed into HTTPretty 267 self.assertEqual(results.rank, "-text_relevance") 268 self.assertEqual(results.match_expression, "Test") 269 270 def test_cloudsearch_results_info(self): 271 """Check num_pages_needed is calculated correctly""" 272 search = SearchConnection(endpoint=HOSTNAME) 273 274 results = search.search(q='Test') 275 276 # This relies on the default response which is fed into HTTPretty 277 self.assertEqual(results.num_pages_needed, 3.0) 278 279 def test_cloudsearch_results_matched(self): 280 """ 281 Check that information objects are passed back through the API 282 correctly. 283 """ 284 search = SearchConnection(endpoint=HOSTNAME) 285 query = search.build_query(q='Test') 286 287 results = search(query) 288 289 self.assertEqual(results.search_service, search) 290 self.assertEqual(results.query, query) 291 292 def test_cloudsearch_results_hits(self): 293 """Check that documents are parsed properly from AWS""" 294 search = SearchConnection(endpoint=HOSTNAME) 295 296 results = search.search(q='Test') 297 298 hits = list(map(lambda x: x['id'], results.docs)) 299 300 # This relies on the default response which is fed into HTTPretty 301 self.assertEqual( 302 hits, ["12341", "12342", "12343", "12344", 303 "12345", "12346", "12347"]) 304 305 def test_cloudsearch_results_iterator(self): 306 """Check the results iterator""" 307 search = SearchConnection(endpoint=HOSTNAME) 308 309 results = search.search(q='Test') 310 results_correct = iter(["12341", "12342", "12343", "12344", 311 "12345", "12346", "12347"]) 312 for x in results: 313 self.assertEqual(x['id'], next(results_correct)) 314 315 316 def test_cloudsearch_results_internal_consistancy(self): 317 """Check the documents length matches the iterator details""" 318 search = SearchConnection(endpoint=HOSTNAME) 319 320 results = search.search(q='Test') 321 322 self.assertEqual(len(results), len(results.docs)) 323 324 def test_cloudsearch_search_nextpage(self): 325 """Check next page query is correct""" 326 search = SearchConnection(endpoint=HOSTNAME) 327 query1 = search.build_query(q='Test') 328 query2 = search.build_query(q='Test') 329 330 results = search(query2) 331 332 self.assertEqual(results.next_page().query.start, 333 query1.start + query1.size) 334 self.assertEqual(query1.q, query2.q) 335 336class CloudSearchSearchFacetTest(CloudSearchSearchBaseTest): 337 response = { 338 'rank': '-text_relevance', 339 'match-expr': "Test", 340 'hits': { 341 'found': 30, 342 'start': 0, 343 'hit': CloudSearchSearchBaseTest.hits 344 }, 345 'info': { 346 'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08', 347 'time-ms': 2, 348 'cpu-time-ms': 0 349 }, 350 'facets': { 351 'tags': {}, 352 'animals': {'constraints': [{'count': '2', 'value': 'fish'}, {'count': '1', 'value': 'lions'}]}, 353 } 354 } 355 356 def test_cloudsearch_search_facets(self): 357 #self.response['facets'] = {'tags': {}} 358 359 search = SearchConnection(endpoint=HOSTNAME) 360 361 results = search.search(q='Test', facet=['tags']) 362 363 self.assertTrue('tags' not in results.facets) 364 self.assertEqual(results.facets['animals'], {u'lions': u'1', u'fish': u'2'}) 365 366 367class CloudSearchNonJsonTest(CloudSearchSearchBaseTest): 368 response = b'<html><body><h1>500 Internal Server Error</h1></body></html>' 369 response_status = 500 370 content_type = 'text/xml' 371 372 def test_response(self): 373 search = SearchConnection(endpoint=HOSTNAME) 374 375 with self.assertRaises(SearchServiceException): 376 search.search(q='Test') 377 378 379class CloudSearchUnauthorizedTest(CloudSearchSearchBaseTest): 380 response = b'<html><body><h1>403 Forbidden</h1>foo bar baz</body></html>' 381 response_status = 403 382 content_type = 'text/html' 383 384 def test_response(self): 385 search = SearchConnection(endpoint=HOSTNAME) 386 387 with self.assertRaisesRegexp(SearchServiceException, 'foo bar baz'): 388 search.search(q='Test') 389 390 391class FakeResponse(object): 392 status_code = 405 393 content = b'' 394 395 396class CloudSearchConnectionTest(unittest.TestCase): 397 cloudsearch = True 398 399 def setUp(self): 400 super(CloudSearchConnectionTest, self).setUp() 401 self.conn = SearchConnection( 402 endpoint='test-domain.cloudsearch.amazonaws.com' 403 ) 404 405 def test_expose_additional_error_info(self): 406 mpo = mock.patch.object 407 fake = FakeResponse() 408 fake.content = b'Nopenopenope' 409 410 # First, in the case of a non-JSON, non-403 error. 411 with mpo(requests, 'get', return_value=fake) as mock_request: 412 with self.assertRaises(SearchServiceException) as cm: 413 self.conn.search(q='not_gonna_happen') 414 415 self.assertTrue('non-json response' in str(cm.exception)) 416 self.assertTrue('Nopenopenope' in str(cm.exception)) 417 418 # Then with JSON & an 'error' key within. 419 fake.content = json.dumps({ 420 'error': "Something went wrong. Oops." 421 }).encode('utf-8') 422 423 with mpo(requests, 'get', return_value=fake) as mock_request: 424 with self.assertRaises(SearchServiceException) as cm: 425 self.conn.search(q='no_luck_here') 426 427 self.assertTrue('Unknown error' in str(cm.exception)) 428 self.assertTrue('went wrong. Oops' in str(cm.exception)) 429