1#!/usr/bin env python 2from boto.cloudsearch2.domain import Domain 3from boto.cloudsearch2.layer1 import CloudSearchConnection 4 5from tests.compat import mock, unittest 6from httpretty import HTTPretty 7 8import json 9 10from boto.cloudsearch2.search import SearchConnection, SearchServiceException 11from boto.compat import six, map 12from tests.unit import AWSMockServiceTestCase 13from tests.unit.cloudsearch2 import DEMO_DOMAIN_DATA 14from tests.unit.cloudsearch2.test_connection import TestCloudSearchCreateDomain 15 16HOSTNAME = "search-demo-userdomain.us-east-1.cloudsearch.amazonaws.com" 17FULL_URL = 'http://%s/2013-01-01/search' % HOSTNAME 18 19 20class CloudSearchSearchBaseTest(unittest.TestCase): 21 22 hits = [ 23 { 24 'id': '12341', 25 'fields': { 26 'title': 'Document 1', 27 'rank': 1 28 } 29 }, 30 { 31 'id': '12342', 32 'fields': { 33 'title': 'Document 2', 34 'rank': 2 35 } 36 }, 37 { 38 'id': '12343', 39 'fields': { 40 'title': 'Document 3', 41 'rank': 3 42 } 43 }, 44 { 45 'id': '12344', 46 'fields': { 47 'title': 'Document 4', 48 'rank': 4 49 } 50 }, 51 { 52 'id': '12345', 53 'fields': { 54 'title': 'Document 5', 55 'rank': 5 56 } 57 }, 58 { 59 'id': '12346', 60 'fields': { 61 'title': 'Document 6', 62 'rank': 6 63 } 64 }, 65 { 66 'id': '12347', 67 'fields': { 68 'title': 'Document 7', 69 'rank': 7 70 } 71 }, 72 ] 73 74 content_type = "text/xml" 75 response_status = 200 76 77 def get_args(self, requestline): 78 (_, request, _) = requestline.split(b" ") 79 (_, request) = request.split(b"?", 1) 80 args = six.moves.urllib.parse.parse_qs(request) 81 return args 82 83 def setUp(self): 84 HTTPretty.enable() 85 body = self.response 86 87 if not isinstance(body, bytes): 88 body = json.dumps(body).encode('utf-8') 89 90 HTTPretty.register_uri(HTTPretty.GET, FULL_URL, 91 body=body, 92 content_type=self.content_type, 93 status=self.response_status) 94 95 def tearDown(self): 96 HTTPretty.disable() 97 98 99class CloudSearchSearchTest(CloudSearchSearchBaseTest): 100 response = { 101 'rank': '-text_relevance', 102 'match-expr': "Test", 103 'hits': { 104 'found': 30, 105 'start': 0, 106 'hit': CloudSearchSearchBaseTest.hits 107 }, 108 'status': { 109 'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08', 110 'time-ms': 2, 111 'cpu-time-ms': 0 112 } 113 114 } 115 116 def test_cloudsearch_qsearch(self): 117 search = SearchConnection(endpoint=HOSTNAME) 118 119 search.search(q='Test', options='TestOptions') 120 121 args = self.get_args(HTTPretty.last_request.raw_requestline) 122 123 self.assertEqual(args[b'q'], [b"Test"]) 124 self.assertEqual(args[b'q.options'], [b"TestOptions"]) 125 self.assertEqual(args[b'start'], [b"0"]) 126 self.assertEqual(args[b'size'], [b"10"]) 127 128 def test_cloudsearch_search_details(self): 129 search = SearchConnection(endpoint=HOSTNAME) 130 131 search.search(q='Test', size=50, start=20) 132 133 args = self.get_args(HTTPretty.last_request.raw_requestline) 134 135 self.assertEqual(args[b'q'], [b"Test"]) 136 self.assertEqual(args[b'size'], [b"50"]) 137 self.assertEqual(args[b'start'], [b"20"]) 138 139 def test_cloudsearch_facet_constraint_single(self): 140 search = SearchConnection(endpoint=HOSTNAME) 141 142 search.search( 143 q='Test', 144 facet={'author': "'John Smith','Mark Smith'"}) 145 146 args = self.get_args(HTTPretty.last_request.raw_requestline) 147 148 self.assertEqual(args[b'facet.author'], 149 [b"'John Smith','Mark Smith'"]) 150 151 def test_cloudsearch_facet_constraint_multiple(self): 152 search = SearchConnection(endpoint=HOSTNAME) 153 154 search.search( 155 q='Test', 156 facet={'author': "'John Smith','Mark Smith'", 157 'category': "'News','Reviews'"}) 158 159 args = self.get_args(HTTPretty.last_request.raw_requestline) 160 161 self.assertEqual(args[b'facet.author'], 162 [b"'John Smith','Mark Smith'"]) 163 self.assertEqual(args[b'facet.category'], 164 [b"'News','Reviews'"]) 165 166 def test_cloudsearch_facet_sort_single(self): 167 search = SearchConnection(endpoint=HOSTNAME) 168 169 search.search(q='Test', facet={'author': {'sort': 'alpha'}}) 170 171 args = self.get_args(HTTPretty.last_request.raw_requestline) 172 173 print(args) 174 175 self.assertEqual(args[b'facet.author'], [b'{"sort": "alpha"}']) 176 177 def test_cloudsearch_facet_sort_multiple(self): 178 search = SearchConnection(endpoint=HOSTNAME) 179 180 search.search(q='Test', facet={'author': {'sort': 'alpha'}, 181 'cat': {'sort': 'count'}}) 182 183 args = self.get_args(HTTPretty.last_request.raw_requestline) 184 185 self.assertEqual(args[b'facet.author'], [b'{"sort": "alpha"}']) 186 self.assertEqual(args[b'facet.cat'], [b'{"sort": "count"}']) 187 188 def test_cloudsearch_result_fields_single(self): 189 search = SearchConnection(endpoint=HOSTNAME) 190 191 search.search(q='Test', return_fields=['author']) 192 193 args = self.get_args(HTTPretty.last_request.raw_requestline) 194 195 self.assertEqual(args[b'return'], [b'author']) 196 197 def test_cloudsearch_result_fields_multiple(self): 198 search = SearchConnection(endpoint=HOSTNAME) 199 200 search.search(q='Test', return_fields=['author', 'title']) 201 202 args = self.get_args(HTTPretty.last_request.raw_requestline) 203 204 self.assertEqual(args[b'return'], [b'author,title']) 205 206 def test_cloudsearch_results_meta(self): 207 """Check returned metadata is parsed correctly""" 208 search = SearchConnection(endpoint=HOSTNAME) 209 210 results = search.search(q='Test') 211 212 # These rely on the default response which is fed into HTTPretty 213 self.assertEqual(results.hits, 30) 214 self.assertEqual(results.docs[0]['fields']['rank'], 1) 215 216 def test_cloudsearch_results_info(self): 217 """Check num_pages_needed is calculated correctly""" 218 search = SearchConnection(endpoint=HOSTNAME) 219 220 results = search.search(q='Test') 221 222 # This relies on the default response which is fed into HTTPretty 223 self.assertEqual(results.num_pages_needed, 3.0) 224 225 def test_cloudsearch_results_matched(self): 226 """ 227 Check that information objects are passed back through the API 228 correctly. 229 """ 230 search = SearchConnection(endpoint=HOSTNAME) 231 query = search.build_query(q='Test') 232 233 results = search(query) 234 235 self.assertEqual(results.search_service, search) 236 self.assertEqual(results.query, query) 237 238 def test_cloudsearch_results_hits(self): 239 """Check that documents are parsed properly from AWS""" 240 search = SearchConnection(endpoint=HOSTNAME) 241 242 results = search.search(q='Test') 243 244 hits = list(map(lambda x: x['id'], results.docs)) 245 246 # This relies on the default response which is fed into HTTPretty 247 self.assertEqual( 248 hits, ["12341", "12342", "12343", "12344", 249 "12345", "12346", "12347"]) 250 251 def test_cloudsearch_results_iterator(self): 252 """Check the results iterator""" 253 search = SearchConnection(endpoint=HOSTNAME) 254 255 results = search.search(q='Test') 256 results_correct = iter(["12341", "12342", "12343", "12344", 257 "12345", "12346", "12347"]) 258 for x in results: 259 self.assertEqual(x['id'], next(results_correct)) 260 261 def test_cloudsearch_results_internal_consistancy(self): 262 """Check the documents length matches the iterator details""" 263 search = SearchConnection(endpoint=HOSTNAME) 264 265 results = search.search(q='Test') 266 267 self.assertEqual(len(results), len(results.docs)) 268 269 def test_cloudsearch_search_nextpage(self): 270 """Check next page query is correct""" 271 search = SearchConnection(endpoint=HOSTNAME) 272 query1 = search.build_query(q='Test') 273 query2 = search.build_query(q='Test') 274 275 results = search(query2) 276 277 self.assertEqual(results.next_page().query.start, 278 query1.start + query1.size) 279 self.assertEqual(query1.q, query2.q) 280 281 282class CloudSearchSearchFacetTest(CloudSearchSearchBaseTest): 283 response = { 284 'rank': '-text_relevance', 285 'match-expr': "Test", 286 'hits': { 287 'found': 30, 288 'start': 0, 289 'hit': CloudSearchSearchBaseTest.hits 290 }, 291 'status': { 292 'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08', 293 'time-ms': 2, 294 'cpu-time-ms': 0 295 }, 296 'facets': { 297 'tags': {}, 298 'animals': {'buckets': [{'count': '2', 'value': 'fish'}, {'count': '1', 'value': 'lions'}]}, 299 } 300 } 301 302 def test_cloudsearch_search_facets(self): 303 #self.response['facets'] = {'tags': {}} 304 305 search = SearchConnection(endpoint=HOSTNAME) 306 307 results = search.search(q='Test', facet={'tags': {}}) 308 309 self.assertTrue('tags' not in results.facets) 310 self.assertEqual(results.facets['animals'], {u'lions': u'1', u'fish': u'2'}) 311 312 313class CloudSearchNonJsonTest(CloudSearchSearchBaseTest): 314 response = b'<html><body><h1>500 Internal Server Error</h1></body></html>' 315 response_status = 500 316 content_type = 'text/xml' 317 318 def test_response(self): 319 search = SearchConnection(endpoint=HOSTNAME) 320 321 with self.assertRaises(SearchServiceException): 322 search.search(q='Test') 323 324 325class CloudSearchUnauthorizedTest(CloudSearchSearchBaseTest): 326 response = b'<html><body><h1>403 Forbidden</h1>foo bar baz</body></html>' 327 response_status = 403 328 content_type = 'text/html' 329 330 def test_response(self): 331 search = SearchConnection(endpoint=HOSTNAME) 332 333 with self.assertRaisesRegexp(SearchServiceException, 'foo bar baz'): 334 search.search(q='Test') 335 336 337class FakeResponse(object): 338 status_code = 405 339 content = b'' 340 341 342class CloudSearchConnectionTest(AWSMockServiceTestCase): 343 cloudsearch = True 344 connection_class = CloudSearchConnection 345 346 def setUp(self): 347 super(CloudSearchConnectionTest, self).setUp() 348 self.conn = SearchConnection( 349 endpoint='test-domain.cloudsearch.amazonaws.com' 350 ) 351 352 def test_expose_additional_error_info(self): 353 mpo = mock.patch.object 354 fake = FakeResponse() 355 fake.content = b'Nopenopenope' 356 357 # First, in the case of a non-JSON, non-403 error. 358 with mpo(self.conn.session, 'get', return_value=fake) as mock_request: 359 with self.assertRaises(SearchServiceException) as cm: 360 self.conn.search(q='not_gonna_happen') 361 362 self.assertTrue('non-json response' in str(cm.exception)) 363 self.assertTrue('Nopenopenope' in str(cm.exception)) 364 365 # Then with JSON & an 'error' key within. 366 fake.content = json.dumps({ 367 'error': "Something went wrong. Oops." 368 }).encode('utf-8') 369 370 with mpo(self.conn.session, 'get', return_value=fake) as mock_request: 371 with self.assertRaises(SearchServiceException) as cm: 372 self.conn.search(q='no_luck_here') 373 374 self.assertTrue('Unknown error' in str(cm.exception)) 375 self.assertTrue('went wrong. Oops' in str(cm.exception)) 376 377 def test_proxy(self): 378 conn = self.service_connection 379 conn.proxy = "127.0.0.1" 380 conn.proxy_user = "john.doe" 381 conn.proxy_pass="p4ssw0rd" 382 conn.proxy_port="8180" 383 conn.use_proxy = True 384 385 domain = Domain(conn, DEMO_DOMAIN_DATA) 386 search = SearchConnection(domain=domain) 387 self.assertEqual(search.session.proxies, {'http': 'http://john.doe:p4ssw0rd@127.0.0.1:8180'}) 388