1#!/usr/bin env python
2
3from tests.compat import mock, unittest
4from httpretty import HTTPretty
5
6import json
7import requests
8
9from boto.cloudsearch.search import SearchConnection, SearchServiceException
10from boto.compat import six, map
11
12HOSTNAME = "search-demo-userdomain.us-east-1.cloudsearch.amazonaws.com"
13FULL_URL = 'http://%s/2011-02-01/search' % HOSTNAME
14
15
16class CloudSearchSearchBaseTest(unittest.TestCase):
17
18    hits = [
19        {
20            'id': '12341',
21            'title': 'Document 1',
22        },
23        {
24            'id': '12342',
25            'title': 'Document 2',
26        },
27        {
28            'id': '12343',
29            'title': 'Document 3',
30        },
31        {
32            'id': '12344',
33            'title': 'Document 4',
34        },
35        {
36            'id': '12345',
37            'title': 'Document 5',
38        },
39        {
40            'id': '12346',
41            'title': 'Document 6',
42        },
43        {
44            'id': '12347',
45            'title': 'Document 7',
46        },
47    ]
48
49    content_type = "text/xml"
50    response_status = 200
51
52    def get_args(self, requestline):
53        (_, request, _) = requestline.split(b" ")
54        (_, request) = request.split(b"?", 1)
55        args = six.moves.urllib.parse.parse_qs(request)
56        return args
57
58    def setUp(self):
59        HTTPretty.enable()
60        body = self.response
61
62        if not isinstance(body, bytes):
63            body = json.dumps(body).encode('utf-8')
64
65        HTTPretty.register_uri(HTTPretty.GET, FULL_URL,
66                               body=body,
67                               content_type=self.content_type,
68                               status=self.response_status)
69
70    def tearDown(self):
71        HTTPretty.disable()
72
73class CloudSearchSearchTest(CloudSearchSearchBaseTest):
74    response = {
75        'rank': '-text_relevance',
76        'match-expr': "Test",
77        'hits': {
78            'found': 30,
79            'start': 0,
80            'hit': CloudSearchSearchBaseTest.hits
81            },
82        'info': {
83            'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08',
84            'time-ms': 2,
85            'cpu-time-ms': 0
86        }
87
88    }
89
90    def test_cloudsearch_qsearch(self):
91        search = SearchConnection(endpoint=HOSTNAME)
92
93        search.search(q='Test')
94
95        args = self.get_args(HTTPretty.last_request.raw_requestline)
96
97        self.assertEqual(args[b'q'], [b"Test"])
98        self.assertEqual(args[b'start'], [b"0"])
99        self.assertEqual(args[b'size'], [b"10"])
100
101    def test_cloudsearch_bqsearch(self):
102        search = SearchConnection(endpoint=HOSTNAME)
103
104        search.search(bq="'Test'")
105
106        args = self.get_args(HTTPretty.last_request.raw_requestline)
107
108        self.assertEqual(args[b'bq'], [b"'Test'"])
109
110    def test_cloudsearch_search_details(self):
111        search = SearchConnection(endpoint=HOSTNAME)
112
113        search.search(q='Test', size=50, start=20)
114
115        args = self.get_args(HTTPretty.last_request.raw_requestline)
116
117        self.assertEqual(args[b'q'], [b"Test"])
118        self.assertEqual(args[b'size'], [b"50"])
119        self.assertEqual(args[b'start'], [b"20"])
120
121    def test_cloudsearch_facet_single(self):
122        search = SearchConnection(endpoint=HOSTNAME)
123
124        search.search(q='Test', facet=["Author"])
125
126        args = self.get_args(HTTPretty.last_request.raw_requestline)
127
128        self.assertEqual(args[b'facet'], [b"Author"])
129
130    def test_cloudsearch_facet_multiple(self):
131        search = SearchConnection(endpoint=HOSTNAME)
132
133        search.search(q='Test', facet=["author", "cat"])
134
135        args = self.get_args(HTTPretty.last_request.raw_requestline)
136
137        self.assertEqual(args[b'facet'], [b"author,cat"])
138
139    def test_cloudsearch_facet_constraint_single(self):
140        search = SearchConnection(endpoint=HOSTNAME)
141
142        search.search(
143            q='Test',
144            facet_constraints={'author': "'John Smith','Mark Smith'"})
145
146        args = self.get_args(HTTPretty.last_request.raw_requestline)
147
148        self.assertEqual(args[b'facet-author-constraints'],
149                         [b"'John Smith','Mark Smith'"])
150
151    def test_cloudsearch_facet_constraint_multiple(self):
152        search = SearchConnection(endpoint=HOSTNAME)
153
154        search.search(
155            q='Test',
156            facet_constraints={'author': "'John Smith','Mark Smith'",
157                               'category': "'News','Reviews'"})
158
159        args = self.get_args(HTTPretty.last_request.raw_requestline)
160
161        self.assertEqual(args[b'facet-author-constraints'],
162                         [b"'John Smith','Mark Smith'"])
163        self.assertEqual(args[b'facet-category-constraints'],
164                         [b"'News','Reviews'"])
165
166    def test_cloudsearch_facet_sort_single(self):
167        search = SearchConnection(endpoint=HOSTNAME)
168
169        search.search(q='Test', facet_sort={'author': 'alpha'})
170
171        args = self.get_args(HTTPretty.last_request.raw_requestline)
172
173        self.assertEqual(args[b'facet-author-sort'], [b'alpha'])
174
175    def test_cloudsearch_facet_sort_multiple(self):
176        search = SearchConnection(endpoint=HOSTNAME)
177
178        search.search(q='Test', facet_sort={'author': 'alpha',
179                                            'cat': 'count'})
180
181        args = self.get_args(HTTPretty.last_request.raw_requestline)
182
183        self.assertEqual(args[b'facet-author-sort'], [b'alpha'])
184        self.assertEqual(args[b'facet-cat-sort'], [b'count'])
185
186    def test_cloudsearch_top_n_single(self):
187        search = SearchConnection(endpoint=HOSTNAME)
188
189        search.search(q='Test', facet_top_n={'author': 5})
190
191        args = self.get_args(HTTPretty.last_request.raw_requestline)
192
193        self.assertEqual(args[b'facet-author-top-n'], [b'5'])
194
195    def test_cloudsearch_top_n_multiple(self):
196        search = SearchConnection(endpoint=HOSTNAME)
197
198        search.search(q='Test', facet_top_n={'author': 5, 'cat': 10})
199
200        args = self.get_args(HTTPretty.last_request.raw_requestline)
201
202        self.assertEqual(args[b'facet-author-top-n'], [b'5'])
203        self.assertEqual(args[b'facet-cat-top-n'], [b'10'])
204
205    def test_cloudsearch_rank_single(self):
206        search = SearchConnection(endpoint=HOSTNAME)
207
208        search.search(q='Test', rank=["date"])
209
210        args = self.get_args(HTTPretty.last_request.raw_requestline)
211
212        self.assertEqual(args[b'rank'], [b'date'])
213
214    def test_cloudsearch_rank_multiple(self):
215        search = SearchConnection(endpoint=HOSTNAME)
216
217        search.search(q='Test', rank=["date", "score"])
218
219        args = self.get_args(HTTPretty.last_request.raw_requestline)
220
221        self.assertEqual(args[b'rank'], [b'date,score'])
222
223    def test_cloudsearch_result_fields_single(self):
224        search = SearchConnection(endpoint=HOSTNAME)
225
226        search.search(q='Test', return_fields=['author'])
227
228        args = self.get_args(HTTPretty.last_request.raw_requestline)
229
230        self.assertEqual(args[b'return-fields'], [b'author'])
231
232    def test_cloudsearch_result_fields_multiple(self):
233        search = SearchConnection(endpoint=HOSTNAME)
234
235        search.search(q='Test', return_fields=['author', 'title'])
236
237        args = self.get_args(HTTPretty.last_request.raw_requestline)
238
239        self.assertEqual(args[b'return-fields'], [b'author,title'])
240
241    def test_cloudsearch_t_field_single(self):
242        search = SearchConnection(endpoint=HOSTNAME)
243
244        search.search(q='Test', t={'year': '2001..2007'})
245
246        args = self.get_args(HTTPretty.last_request.raw_requestline)
247
248        self.assertEqual(args[b't-year'], [b'2001..2007'])
249
250    def test_cloudsearch_t_field_multiple(self):
251        search = SearchConnection(endpoint=HOSTNAME)
252
253        search.search(q='Test', t={'year': '2001..2007', 'score': '10..50'})
254
255        args = self.get_args(HTTPretty.last_request.raw_requestline)
256
257        self.assertEqual(args[b't-year'], [b'2001..2007'])
258        self.assertEqual(args[b't-score'], [b'10..50'])
259
260    def test_cloudsearch_results_meta(self):
261        """Check returned metadata is parsed correctly"""
262        search = SearchConnection(endpoint=HOSTNAME)
263
264        results = search.search(q='Test')
265
266        # These rely on the default response which is fed into HTTPretty
267        self.assertEqual(results.rank, "-text_relevance")
268        self.assertEqual(results.match_expression, "Test")
269
270    def test_cloudsearch_results_info(self):
271        """Check num_pages_needed is calculated correctly"""
272        search = SearchConnection(endpoint=HOSTNAME)
273
274        results = search.search(q='Test')
275
276        # This relies on the default response which is fed into HTTPretty
277        self.assertEqual(results.num_pages_needed, 3.0)
278
279    def test_cloudsearch_results_matched(self):
280        """
281        Check that information objects are passed back through the API
282        correctly.
283        """
284        search = SearchConnection(endpoint=HOSTNAME)
285        query = search.build_query(q='Test')
286
287        results = search(query)
288
289        self.assertEqual(results.search_service, search)
290        self.assertEqual(results.query, query)
291
292    def test_cloudsearch_results_hits(self):
293        """Check that documents are parsed properly from AWS"""
294        search = SearchConnection(endpoint=HOSTNAME)
295
296        results = search.search(q='Test')
297
298        hits = list(map(lambda x: x['id'], results.docs))
299
300        # This relies on the default response which is fed into HTTPretty
301        self.assertEqual(
302            hits, ["12341", "12342", "12343", "12344",
303                   "12345", "12346", "12347"])
304
305    def test_cloudsearch_results_iterator(self):
306        """Check the results iterator"""
307        search = SearchConnection(endpoint=HOSTNAME)
308
309        results = search.search(q='Test')
310        results_correct = iter(["12341", "12342", "12343", "12344",
311                                "12345", "12346", "12347"])
312        for x in results:
313            self.assertEqual(x['id'], next(results_correct))
314
315
316    def test_cloudsearch_results_internal_consistancy(self):
317        """Check the documents length matches the iterator details"""
318        search = SearchConnection(endpoint=HOSTNAME)
319
320        results = search.search(q='Test')
321
322        self.assertEqual(len(results), len(results.docs))
323
324    def test_cloudsearch_search_nextpage(self):
325        """Check next page query is correct"""
326        search = SearchConnection(endpoint=HOSTNAME)
327        query1 = search.build_query(q='Test')
328        query2 = search.build_query(q='Test')
329
330        results = search(query2)
331
332        self.assertEqual(results.next_page().query.start,
333                         query1.start + query1.size)
334        self.assertEqual(query1.q, query2.q)
335
336class CloudSearchSearchFacetTest(CloudSearchSearchBaseTest):
337    response = {
338        'rank': '-text_relevance',
339        'match-expr': "Test",
340        'hits': {
341            'found': 30,
342            'start': 0,
343            'hit': CloudSearchSearchBaseTest.hits
344            },
345        'info': {
346            'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08',
347            'time-ms': 2,
348            'cpu-time-ms': 0
349        },
350        'facets': {
351            'tags': {},
352            'animals': {'constraints': [{'count': '2', 'value': 'fish'}, {'count': '1', 'value': 'lions'}]},
353        }
354    }
355
356    def test_cloudsearch_search_facets(self):
357        #self.response['facets'] = {'tags': {}}
358
359        search = SearchConnection(endpoint=HOSTNAME)
360
361        results = search.search(q='Test', facet=['tags'])
362
363        self.assertTrue('tags' not in results.facets)
364        self.assertEqual(results.facets['animals'], {u'lions': u'1', u'fish': u'2'})
365
366
367class CloudSearchNonJsonTest(CloudSearchSearchBaseTest):
368    response = b'<html><body><h1>500 Internal Server Error</h1></body></html>'
369    response_status = 500
370    content_type = 'text/xml'
371
372    def test_response(self):
373        search = SearchConnection(endpoint=HOSTNAME)
374
375        with self.assertRaises(SearchServiceException):
376            search.search(q='Test')
377
378
379class CloudSearchUnauthorizedTest(CloudSearchSearchBaseTest):
380    response = b'<html><body><h1>403 Forbidden</h1>foo bar baz</body></html>'
381    response_status = 403
382    content_type = 'text/html'
383
384    def test_response(self):
385        search = SearchConnection(endpoint=HOSTNAME)
386
387        with self.assertRaisesRegexp(SearchServiceException, 'foo bar baz'):
388            search.search(q='Test')
389
390
391class FakeResponse(object):
392    status_code = 405
393    content = b''
394
395
396class CloudSearchConnectionTest(unittest.TestCase):
397    cloudsearch = True
398
399    def setUp(self):
400        super(CloudSearchConnectionTest, self).setUp()
401        self.conn = SearchConnection(
402            endpoint='test-domain.cloudsearch.amazonaws.com'
403        )
404
405    def test_expose_additional_error_info(self):
406        mpo = mock.patch.object
407        fake = FakeResponse()
408        fake.content = b'Nopenopenope'
409
410        # First, in the case of a non-JSON, non-403 error.
411        with mpo(requests, 'get', return_value=fake) as mock_request:
412            with self.assertRaises(SearchServiceException) as cm:
413                self.conn.search(q='not_gonna_happen')
414
415            self.assertTrue('non-json response' in str(cm.exception))
416            self.assertTrue('Nopenopenope' in str(cm.exception))
417
418        # Then with JSON & an 'error' key within.
419        fake.content = json.dumps({
420            'error': "Something went wrong. Oops."
421        }).encode('utf-8')
422
423        with mpo(requests, 'get', return_value=fake) as mock_request:
424            with self.assertRaises(SearchServiceException) as cm:
425                self.conn.search(q='no_luck_here')
426
427            self.assertTrue('Unknown error' in str(cm.exception))
428            self.assertTrue('went wrong. Oops' in str(cm.exception))
429