[ckan-changes] commit/ckan: 5 new changesets
Bitbucket
commits-noreply at bitbucket.org
Fri Sep 30 15:00:20 UTC 2011
5 new changesets in ckan:
http://bitbucket.org/okfn/ckan/changeset/615d42dbb9d2/
changeset: 615d42dbb9d2
branch: defect-1365-search-params
user: dread
date: 2011-09-29 16:57:29
summary: [lib]: Completed translation of legacy search params to solr format. Added back all old tests. Added basic functional test for search controller.
affected #: 9 files (-1 bytes)
--- a/ckan/config/routing.py Wed Sep 28 19:06:06 2011 +0100
+++ b/ckan/config/routing.py Thu Sep 29 15:57:29 2011 +0100
@@ -93,7 +93,7 @@
requirements=dict(register=register_list_str),
conditions=dict(method=['DELETE']))
map.connect('/api/{ver:3}/action/{logic_function}', controller='api', action='action',
- conditions=dict(method=['GET']))
+ conditions=dict(method=['GET', 'POST']))
map.connect('/api/{ver:1|2}/qos/throughput/',
controller='api', action='throughput',
requirements=dict(register=register_list_str),
@@ -110,7 +110,7 @@
map.connect('/api/rest', controller='api', action='index')
map.connect('/api/action/{logic_function}', controller='api', action='action',
- conditions=dict(method=['GET']))
+ conditions=dict(method=['GET', 'POST']))
map.connect('/api/rest/{register}', controller='api', action='list',
requirements=dict(register=register_list_str),
conditions=dict(method=['GET'])
--- a/ckan/controllers/api.py Wed Sep 28 19:06:06 2011 +0100
+++ b/ckan/controllers/api.py Thu Sep 29 15:57:29 2011 +0100
@@ -387,6 +387,7 @@
def search(self, ver=None, register=None):
log.debug('search %s params: %r' % (register, request.params))
+ ver = ver or '1' # i.e. default to v1
if register == 'revision':
since_time = None
if request.params.has_key('since_id'):
--- a/ckan/lib/search/common.py Wed Sep 28 19:06:06 2011 +0100
+++ b/ckan/lib/search/common.py Thu Sep 29 15:57:29 2011 +0100
@@ -4,6 +4,7 @@
class SearchIndexError(Exception): pass
class SearchError(Exception): pass
+class SearchQueryError(SearchError): pass
DEFAULT_SOLR_URL = 'http://127.0.0.1:8983/solr'
--- a/ckan/lib/search/query.py Wed Sep 28 19:06:06 2011 +0100
+++ b/ckan/lib/search/query.py Thu Sep 29 15:57:29 2011 +0100
@@ -1,10 +1,10 @@
-import json
from pylons import config
from paste.deploy.converters import asbool
from paste.util.multidict import MultiDict
from ckan import model
from ckan.logic import get_action
-from common import make_connection, SearchError
+from ckan.lib.helpers import json
+from common import make_connection, SearchError, SearchQueryError
import logging
log = logging.getLogger(__name__)
@@ -25,7 +25,7 @@
support, so use this function to convert those to SOLR syntax.
See tests for examples.
- raises ValueError on invalid params.
+ raises SearchQueryError on invalid params.
'''
options = QueryOptions(**legacy_params)
options.validate()
@@ -35,7 +35,8 @@
solr_q_list.append(solr_params['q'].replace('+', ' '))
non_solr_params = set(legacy_params.keys()) - VALID_SOLR_PARAMETERS
for search_key in non_solr_params:
- value = str(legacy_params[search_key]).replace('+', ' ')
+ value_obj = legacy_params[search_key]
+ value = str(value_obj).replace('+', ' ')
if search_key == 'all_fields':
if value:
solr_params['fl'] = '*'
@@ -45,6 +46,14 @@
solr_params['rows'] = value
elif search_key == 'order_by':
solr_params['sort'] = '%s asc' % value
+ elif search_key == 'tags':
+ if isinstance(value_obj, list):
+ tag_list = value_obj
+ elif isinstance(value_obj, basestring):
+ tag_list = [value_obj]
+ else:
+ raise SearchQueryError('Was expecting either a string or JSON list for the tags parameter: %r' % value)
+ solr_q_list.extend(['tags:%s' % tag for tag in tag_list])
else:
if ' ' in value:
value = '"%s"' % value
@@ -83,14 +92,14 @@
try:
value = asbool(value)
except ValueError:
- raise SearchError('Value for search option %r must be True or False (1 or 0) but received %r' % (key, value))
+ raise SearchQueryError('Value for search option %r must be True or False (1 or 0) but received %r' % (key, value))
elif key in self.INTEGER_OPTIONS:
try:
value = int(value)
except ValueError:
- raise SearchError('Value for search option %r must be an integer but received %r' % (key, value))
+ raise SearchQueryError('Value for search option %r must be an integer but received %r' % (key, value))
elif key in self.UNSUPPORTED_OPTIONS:
- raise SearchError('Search option %r is not supported' % key)
+ raise SearchQueryError('Search option %r is not supported' % key)
self[key] = value
def __getattr__(self, name):
@@ -218,7 +227,7 @@
# check that query keys are valid
if not set(query.keys()) <= VALID_SOLR_PARAMETERS:
invalid_params = [s for s in set(query.keys()) - VALID_SOLR_PARAMETERS]
- raise SearchError("Invalid search parameters: %s" % invalid_params)
+ raise SearchQueryError("Invalid search parameters: %s" % invalid_params)
# default query is to return all documents
q = query.get('q')
--- a/ckan/tests/functional/api/base.py Wed Sep 28 19:06:06 2011 +0100
+++ b/ckan/tests/functional/api/base.py Thu Sep 29 15:57:29 2011 +0100
@@ -32,6 +32,9 @@
api_version = None
+ ref_package_by = ''
+ ref_group_by = ''
+
def get(self, offset, status=[200]):
response = self.app.get(offset, status=status,
extra_environ=self.get_extra_environ())
@@ -117,6 +120,24 @@
def data_from_res(self, res):
return self.loads(res.body)
+ def package_ref_from_name(self, package_name):
+ package = self.get_package_by_name(unicode(package_name))
+ if package is None:
+ return package_name
+ else:
+ return self.ref_package(package)
+
+ def package_id_from_ref(self, package_name):
+ package = self.get_package_by_name(unicode(package_name))
+ if package is None:
+ return package_name
+ else:
+ return self.ref_package(package)
+
+ def ref_package(self, package):
+ assert self.ref_package_by in ['id', 'name']
+ return getattr(package, self.ref_package_by)
+
def get_expected_api_version(self):
return self.api_version
@@ -142,11 +163,7 @@
class Api1and2TestCase(object):
''' Utils for v1 and v2 API.
* RESTful URL utils
- * handling of different ways of referencing packages (by name or id)
'''
- ref_package_by = ''
- ref_group_by = ''
-
def package_offset(self, package_name=None):
if package_name is None:
# Package Register
@@ -156,24 +173,6 @@
package_ref = self.package_ref_from_name(package_name)
return self.offset('/rest/dataset/%s' % package_ref)
- def package_ref_from_name(self, package_name):
- package = self.get_package_by_name(unicode(package_name))
- if package is None:
- return package_name
- else:
- return self.ref_package(package)
-
- def package_id_from_ref(self, package_name):
- package = self.get_package_by_name(unicode(package_name))
- if package is None:
- return package_name
- else:
- return self.ref_package(package)
-
- def ref_package(self, package):
- assert self.ref_package_by in ['id', 'name']
- return getattr(package, self.ref_package_by)
-
def group_offset(self, group_name=None):
if group_name is None:
# Group Register
@@ -303,9 +302,9 @@
class Api3TestCase(ApiTestCase):
api_version = '3'
- ref_package_by = 'id'
- ref_group_by = 'id'
- ref_tag_by = 'id'
+ ref_package_by = 'name'
+ ref_group_by = 'name'
+ ref_tag_by = 'name'
def assert_msg_represents_anna(self, msg):
super(Api2TestCase, self).assert_msg_represents_anna(msg)
--- a/ckan/tests/functional/api/test_package_search.py Wed Sep 28 19:06:06 2011 +0100
+++ b/ckan/tests/functional/api/test_package_search.py Thu Sep 29 15:57:29 2011 +0100
@@ -39,10 +39,6 @@
for expected_package_name in expected_package_names]
assert_equal(set(res_dict['results']), set(expected_pkgs))
- def package_ref_from_name(self, package_name):
- package = self.get_package_by_name(package_name)
- return self.ref_package(package)
-
def test_00_read_search_params(self):
def check(request_params, expected_params):
params = ApiController._get_search_params(request_params)
@@ -50,8 +46,8 @@
# uri parameters
check(UnicodeMultiDict({'q': '', 'ref': 'boris'}),
{"q": "", "ref": "boris"})
- check(UnicodeMultiDict({}),
- {})
+ check(UnicodeMultiDict({'filter_by_openness': '1'}),
+ {'filter_by_openness': '1'})
# uri json
check(UnicodeMultiDict({'qjson': '{"q": "", "ref": "boris"}'}),
{"q": "", "ref": "boris"})
@@ -76,7 +72,6 @@
offset = self.base_url + '?q=%s' % self.package_fixture_data['name']
res = self.app.get(offset, status=200)
res_dict = self.data_from_res(res)
- print res_dict
self.assert_results(res_dict, ['testpkg'])
assert res_dict['count'] == 1, res_dict['count']
@@ -132,53 +127,11 @@
self.assert_results(res_dict, [u'annakarenina'])
assert res_dict['count'] == 1, res_dict['count']
- def test_07_uri_qjson_tags(self):
- query = {'q': 'tags:tolstoy'}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_results(res_dict, [u'annakarenina'])
- assert res_dict['count'] == 1, res_dict
-
- def test_07_uri_qjson_tags_multiple(self):
- query = {'q': 'tags:tolstoy tags:russian'}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- print offset
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_results(res_dict, [u'annakarenina'])
- assert res_dict['count'] == 1, res_dict
-
- def test_07_uri_qjson_tags_reverse(self):
- query = {'q': 'tags:russian'}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_results(res_dict, [u'annakarenina', u'warandpeace'])
- assert res_dict['count'] == 2, res_dict
-
- def test_07_uri_qjson_extras_2(self):
- query = {'q': "national_statistic:yes"}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_results(res_dict, ['testpkg'])
- assert res_dict['count'] == 1, res_dict
-
def test_08_uri_qjson_malformed(self):
offset = self.base_url + '?qjson="q":""' # user forgot the curly braces
res = self.app.get(offset, status=400)
self.assert_json_response(res, 'Bad request - Could not read parameters')
- def test_08_all_fields_syntax_error(self):
- offset = self.base_url + '?all_fields=should_be_boolean' # invalid all_fields value
- res = self.app.get(offset, status=400)
- assert('all_fields' in res.body)
-
def test_09_just_tags(self):
offset = self.base_url + '?q=tags:russian'
res = self.app.get(offset, status=200)
@@ -211,6 +164,10 @@
res_dict = self.data_from_res(res)
assert_equal(res_dict['count'], 3)
+## filter_by_openness and filter_by_downloadable funcionality
+## to be removed for ckan v1.5 (even for v1 and v2 API)
+## see #1360
+
def test_12_filter_by_openness_qjson(self):
query = {'q': '', 'filter_by_openness': '1'}
json_query = self.dumps(query)
@@ -221,13 +178,23 @@
offset = self.base_url + '?filter_by_openness=1'
res = self.app.get(offset, status=400)
- def test_13_just_groups(self):
- offset = self.base_url + '?q=groups:roger'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
+## def test_12_filter_by_openness_off_qjson(self):
+## query = {'q': '', 'filter_by_openness': '0'}
+## json_query = self.dumps(query)
+## offset = self.base_url + '?qjson=%s' % json_query
+## res = self.app.get(offset, status=200)
+## res_dict = self.data_from_res(res)
+## assert_equal(res_dict['count'], 3)
+
+## def test_12_filter_by_openness_off_q(self):
+## offset = self.base_url + '?filter_by_openness=0'
+## res = self.app.get(offset, status=200)
+## res_dict = self.data_from_res(res)
+## assert_equal(res_dict['count'], 3)
class LegacyOptionsTestCase(ApiTestCase, ControllerTestCase):
+ '''Here are tests with URIs in the syntax they were in
+ for API v1 and v2.'''
@classmethod
def setup_class(self):
setup_test_search_index()
@@ -251,6 +218,34 @@
model.repo.rebuild_db()
search.clear()
+ def test_07_uri_qjson_tags(self):
+ query = {'q': '', 'tags':['tolstoy']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_tags_multiple(self):
+ query = {'q': '', 'tags':['tolstoy', 'russian']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ print offset
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_tags_reverse(self):
+ query = {'q': '', 'tags':['russian']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina', u'warandpeace'])
+ assert res_dict['count'] == 2, res_dict
+
def test_07_uri_qjson_extras(self):
# TODO: solr is not currently set up to allow partial matches
# and extras are not saved as multivalued so this
@@ -265,6 +260,15 @@
self.assert_results(res_dict, ['testpkg'])
assert res_dict['count'] == 1, res_dict
+ def test_07_uri_qjson_extras_2(self):
+ query = {"national_statistic":"yes"}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict
+
def test_08_all_fields(self):
rating = model.Rating(user_ip_address=u'123.1.2.3',
package=self.anna,
@@ -272,7 +276,7 @@
model.Session.add(rating)
model.repo.commit_and_remove()
- query = {'q': 'russian', 'all_fields': '1'}
+ query = {'q': 'russian', 'all_fields': 1}
json_query = self.dumps(query)
offset = self.base_url + '?qjson=%s' % json_query
res = self.app.get(offset, status=200)
@@ -294,12 +298,25 @@
res2 = self.app.get(offset, status=200)
assert_equal(res2.body, res.body)
- def test_09_tags(self):
+ def test_08_all_fields_syntax_error(self):
+ offset = self.base_url + '?all_fields=should_be_boolean' # invalid all_fields value
+ res = self.app.get(offset, status=400)
+ assert('boolean' in res.body)
+ assert('all_fields' in res.body)
+ self.assert_json_response(res, 'boolean')
+
+ def test_09_just_tags(self):
offset = self.base_url + '?tags=tolstoy'
res = self.app.get(offset, status=200)
res_dict = self.data_from_res(res)
assert res_dict['count'] == 1, res_dict
+ def test_10_multiple_tags_with_plus(self):
+ offset = self.base_url + '?tags=tolstoy+russian&all_fields=1'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
def test_10_multiple_tags_with_ampersand(self):
offset = self.base_url + '?tags=tolstoy&tags=russian&all_fields=1'
res = self.app.get(offset, status=200)
@@ -334,11 +351,55 @@
print res.body
assert('should_be_integer' in res.body)
+ def test_13_just_groups(self):
+ offset = self.base_url + '?groups=roger'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
class TestPackageSearchApi1(Api1TestCase, PackageSearchApiTestCase,
LegacyOptionsTestCase): pass
class TestPackageSearchApi2(Api2TestCase, PackageSearchApiTestCase,
LegacyOptionsTestCase): pass
class TestPackageSearchApi3(Api3TestCase, PackageSearchApiTestCase):
+ '''Here are tests with URIs in specifically SOLR syntax.'''
+ def test_07_uri_qjson_tags(self):
+ query = {'q': 'tags:tolstoy'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_tags_multiple(self):
+ query = {'q': 'tags:tolstoy tags:russian'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ print offset
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_tags_reverse(self):
+ query = {'q': 'tags:russian'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina', u'warandpeace'])
+ assert res_dict['count'] == 2, res_dict
+
+ def test_07_uri_qjson_extras_2(self):
+ query = {'q': "national_statistic:yes"}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict
+
def test_08_all_fields(self):
query = {'q': 'russian', 'fl': '*'}
json_query = self.dumps(query)
@@ -376,6 +437,32 @@
assert len(res_dict['results']) == 1, res_dict
assert res_dict['results'][0]['name'] == 'annakarenina', res_dict['results'][0]['name']
+ def test_11_pagination_offset_limit(self):
+ offset = self.base_url + '?fl=*&q=tags:russian&offset=1&limit=1&order_by=name'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ assert len(res_dict['results']) == 1, res_dict
+ assert res_dict['results'][0]['name'] == 'warandpeace', res_dict['results'][0]['name']
+
+ def test_11_pagination_syntax_error(self):
+ offset = self.base_url + '?fl=*&q="tags:russian"&start=should_be_integer&rows=1&sort=name' # invalid offset value
+ res = self.app.get(offset, status=400)
+ print res.body
+ assert('should_be_integer' in res.body)
+
+ def test_12_v1_or_v2_syntax(self):
+ offset = self.base_url + '?all_fields=1'
+ res = self.app.get(offset, status=400)
+ assert("Invalid search parameters: ['all_fields']" in res.body), res.body
+
+ def test_13_just_groups(self):
+ offset = self.base_url + '?q=groups:roger'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+
class TestPackageSearchApiUnversioned(PackageSearchApiTestCase,
ApiUnversionedTestCase,
LegacyOptionsTestCase): pass
--- a/ckan/tests/functional/api/test_resource_search.py Wed Sep 28 19:06:06 2011 +0100
+++ b/ckan/tests/functional/api/test_resource_search.py Thu Sep 29 15:57:29 2011 +0100
@@ -79,7 +79,7 @@
assert 'package_id' in result.body, result.body
-class TestResourceSearchApi1(ResourceSearchApiTestCase, Api1TestCase): pass
+class TestResourceSearchApi1(Api1TestCase, ResourceSearchApiTestCase): pass
class TestResourceSearchApi2(Api2TestCase, ResourceSearchApiTestCase): pass
class TestResourceSearchApiUnversioned(ApiUnversionedTestCase, ResourceSearchApiTestCase):
pass
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckan/tests/functional/test_search.py Thu Sep 29 15:57:29 2011 +0100
@@ -0,0 +1,68 @@
+# These only test that the controller is passing on queries correctly
+# to the search library. The search library is tested in:
+# ckan/tests/lib/test_solr_package_search.py
+
+import re
+from nose.tools import assert_equal
+
+from ckan.tests import (TestController, CreateTestData,
+ setup_test_search_index, html_check)
+from ckan import model
+import ckan.lib.search as search
+
+class TestSearch(TestController, html_check.HtmlCheckMethods):
+ # 'penguin' is in all test search packages
+ q_all = u'penguin'
+
+ @classmethod
+ def setup_class(cls):
+ model.Session.remove()
+ setup_test_search_index()
+ CreateTestData.create_search_test_data()
+ cls.count_re = re.compile('<strong>(\d)</strong> datasets found')
+
+ @classmethod
+ def teardown_class(cls):
+ model.repo.rebuild_db()
+ search.clear()
+
+ def _pkg_names(self, result):
+ return ' '.join(result['results'])
+
+ def _check_results(self, res, expected_count, expected_package_names=[]):
+ '''Takes a search result web page and determines whether the
+ search results displayed match the expected count and names
+ of packages.'''
+ # get count
+ content = self.named_div('content', res)
+ count_match = self.count_re.search(content)
+ assert count_match
+ assert_equal(len(count_match.groups()), 1)
+ count = int(count_match.groups()[0])
+ assert_equal(count, expected_count)
+
+ # check package names
+ if isinstance(expected_package_names, basestring):
+ expected_package_names = [expected_package_names]
+ for expected_name in expected_package_names:
+ expected_html = '<a href="/dataset/%s">' % expected_name
+ assert expected_html in res.body, \
+ 'Could not find package name %r in the results page'
+
+ def test_1_all_records(self):
+ res = self.app.get('/dataset?q')
+ result = self._check_results(res, 6, 'gils')
+
+ def test_1_name(self):
+ # exact name
+ res = self.app.get('/dataset?q=gils')
+ result = self._check_results(res, 1, 'gils')
+
+ def test_2_title(self):
+ # exact title, one word
+ res = self.app.get('/dataset?q=Opengov.se')
+ result = self._check_results(res, 1, 'se-opengov')
+
+ # multiple words
+ res = self.app.get('/dataset?q=Government%20Expenditure')
+ result = self._check_results(res, 1, 'uk-government-expenditure')
--- a/ckan/tests/lib/test_solr_package_search.py Wed Sep 28 19:06:06 2011 +0100
+++ b/ckan/tests/lib/test_solr_package_search.py Thu Sep 29 15:57:29 2011 +0100
@@ -19,6 +19,10 @@
assert_raises(search.SearchError, convert, {'title': 'bob', 'all_fields': 'non-boolean'})
assert_equal(convert({'q': 'bob', 'order_by': 'name'}), {'q': 'bob', 'sort':'name asc'})
assert_equal(convert({'q': 'bob', 'offset': '0', 'limit': '10'}), {'q': 'bob', 'start':'0', 'rows':'10'})
+ assert_equal(convert({'tags': ['russian', 'tolstoy']}), {'q': 'tags:russian tags:tolstoy'})
+ assert_equal(convert({'tags': ['tolstoy']}), {'q': 'tags:tolstoy'})
+ assert_raises(search.SearchError, convert, {'tags': 'tolstoy'})
+ assert_raises(search.SearchError, convert, {'tags': {'tolstoy':1}})
class TestSearch(TestController):
# 'penguin' is in all test search packages
@@ -270,11 +274,9 @@
model.repo.rebuild_db()
search.clear()
- def _check_search_results(self, terms, expected_count, expected_packages=[], only_open=False, only_downloadable=False):
+ def _check_search_results(self, terms, expected_count, expected_packages=[]):
query = {
'q': unicode(terms),
- 'filter_by_openness': only_open,
- 'filter_by_downloadable': only_downloadable
}
result = search.query_for(model.Package).run(query)
pkgs = result['results']
http://bitbucket.org/okfn/ckan/changeset/5b20ae1673ea/
changeset: 5b20ae1673ea
branch: defect-1365-search-params
user: dread
date: 2011-09-29 18:21:54
summary: [lib/search]: #1360 Dropped filter_by_downloadable/openness for searching.
affected #: 4 files (-1 bytes)
--- a/ckan/lib/search/query.py Thu Sep 29 15:57:29 2011 +0100
+++ b/ckan/lib/search/query.py Thu Sep 29 17:21:54 2011 +0100
@@ -12,7 +12,6 @@
VALID_SOLR_PARAMETERS = set([
'q', 'fl', 'fq', 'rows', 'sort', 'start', 'wt', 'qf',
- 'filter_by_downloadable', 'filter_by_openness',
'facet', 'facet.mincount', 'facet.limit', 'facet.field'
])
@@ -70,6 +69,8 @@
"""
Options specify aspects of the search query which are only tangentially related
to the query terms (such as limits, etc.).
+ NB This is used only by legacy package search and current resource & tag search.
+ Modern SOLR package search leaves this to SOLR syntax.
"""
BOOLEAN_OPTIONS = ['all_fields']
@@ -263,14 +264,6 @@
# return results as json encoded string
query['wt'] = query.get('wt', 'json')
- # check if filtering by downloadable or open license
- if int(query.get('filter_by_downloadable', 0)):
- query['fq'] += u" +res_url:[* TO *] " # not null resource URL
- if int(query.get('filter_by_openness', 0)):
- licenses = ["license_id:%s" % id for id in self.open_licenses]
- licenses = " OR ".join(licenses)
- query['fq'] += " +(%s) " % licenses
-
# query field weighting: disabled for now as solr 3.* is required for
# the 'edismax' query parser, our current Ubuntu version only has
# packages for 1.4
--- a/ckan/tests/functional/api/test_misc.py Thu Sep 29 15:57:29 2011 +0100
+++ b/ckan/tests/functional/api/test_misc.py Thu Sep 29 17:21:54 2011 +0100
@@ -42,5 +42,5 @@
class TestQosApi1(Api1TestCase, QosApiTestCase): pass
class TestMiscApi2(Api2TestCase, MiscApiTestCase): pass
class TestQosApi2(Api2TestCase, QosApiTestCase): pass
-class TestMiscApiUnversioned(MiscApiTestCase, ApiUnversionedTestCase): pass
+class TestMiscApiUnversioned(ApiUnversionedTestCase, MiscApiTestCase): pass
class TestQosApiUnversioned(ApiUnversionedTestCase, QosApiTestCase): pass
--- a/ckan/tests/functional/api/test_package_search.py Thu Sep 29 15:57:29 2011 +0100
+++ b/ckan/tests/functional/api/test_package_search.py Thu Sep 29 17:21:54 2011 +0100
@@ -46,8 +46,6 @@
# uri parameters
check(UnicodeMultiDict({'q': '', 'ref': 'boris'}),
{"q": "", "ref": "boris"})
- check(UnicodeMultiDict({'filter_by_openness': '1'}),
- {'filter_by_openness': '1'})
# uri json
check(UnicodeMultiDict({'qjson': '{"q": "", "ref": "boris"}'}),
{"q": "", "ref": "boris"})
@@ -164,33 +162,16 @@
res_dict = self.data_from_res(res)
assert_equal(res_dict['count'], 3)
-## filter_by_openness and filter_by_downloadable funcionality
-## to be removed for ckan v1.5 (even for v1 and v2 API)
-## see #1360
+ def test_12_filter_by_openness(self):
+ offset = self.base_url + '?filter_by_openness=1'
+ res = self.app.get(offset, status=400) # feature dropped in #1360
+ assert "'filter_by_openness'" in res.body, res.body
- def test_12_filter_by_openness_qjson(self):
- query = {'q': '', 'filter_by_openness': '1'}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=400)
+ def test_12_filter_by_downloadable(self):
+ offset = self.base_url + '?filter_by_downloadable=1'
+ res = self.app.get(offset, status=400) # feature dropped in #1360
+ assert "'filter_by_downloadable'" in res.body, res.body
- def test_12_filter_by_openness_q(self):
- offset = self.base_url + '?filter_by_openness=1'
- res = self.app.get(offset, status=400)
-
-## def test_12_filter_by_openness_off_qjson(self):
-## query = {'q': '', 'filter_by_openness': '0'}
-## json_query = self.dumps(query)
-## offset = self.base_url + '?qjson=%s' % json_query
-## res = self.app.get(offset, status=200)
-## res_dict = self.data_from_res(res)
-## assert_equal(res_dict['count'], 3)
-
-## def test_12_filter_by_openness_off_q(self):
-## offset = self.base_url + '?filter_by_openness=0'
-## res = self.app.get(offset, status=200)
-## res_dict = self.data_from_res(res)
-## assert_equal(res_dict['count'], 3)
class LegacyOptionsTestCase(ApiTestCase, ControllerTestCase):
'''Here are tests with URIs in the syntax they were in
@@ -438,7 +419,7 @@
assert res_dict['results'][0]['name'] == 'annakarenina', res_dict['results'][0]['name']
def test_11_pagination_offset_limit(self):
- offset = self.base_url + '?fl=*&q=tags:russian&offset=1&limit=1&order_by=name'
+ offset = self.base_url + '?fl=*&q=tags:russian&start=1&rows=1&sort=name asc'
res = self.app.get(offset, status=200)
res_dict = self.data_from_res(res)
assert res_dict['count'] == 2, res_dict
@@ -446,7 +427,7 @@
assert res_dict['results'][0]['name'] == 'warandpeace', res_dict['results'][0]['name']
def test_11_pagination_syntax_error(self):
- offset = self.base_url + '?fl=*&q="tags:russian"&start=should_be_integer&rows=1&sort=name' # invalid offset value
+ offset = self.base_url + '?fl=*&q=tags:russian&start=should_be_integer&rows=1&sort=name asc' # invalid offset value
res = self.app.get(offset, status=400)
print res.body
assert('should_be_integer' in res.body)
--- a/ckan/tests/lib/test_solr_package_search.py Thu Sep 29 15:57:29 2011 +0100
+++ b/ckan/tests/lib/test_solr_package_search.py Thu Sep 29 17:21:54 2011 +0100
@@ -21,7 +21,7 @@
assert_equal(convert({'q': 'bob', 'offset': '0', 'limit': '10'}), {'q': 'bob', 'start':'0', 'rows':'10'})
assert_equal(convert({'tags': ['russian', 'tolstoy']}), {'q': 'tags:russian tags:tolstoy'})
assert_equal(convert({'tags': ['tolstoy']}), {'q': 'tags:tolstoy'})
- assert_raises(search.SearchError, convert, {'tags': 'tolstoy'})
+ assert_equal(convert({'tags': 'tolstoy'}), {'q': 'tags:tolstoy'})
assert_raises(search.SearchError, convert, {'tags': {'tolstoy':1}})
class TestSearch(TestController):
@@ -296,9 +296,6 @@
self._check_search_results('groups:david', 2)
self._check_search_results('groups:roger', 1)
self._check_search_results('groups:lenny', 0)
- self._check_search_results('annakarenina', 1, ['annakarenina'], True, False)
- self._check_search_results('annakarenina', 1, ['annakarenina'], False, True)
- self._check_search_results('annakarenina', 1, ['annakarenina'], True, True)
class TestGeographicCoverage(TestController):
http://bitbucket.org/okfn/ckan/changeset/825c4c43eafe/
changeset: 825c4c43eafe
branch: defect-1365-search-params
user: dread
date: 2011-09-30 16:55:21
summary: [controllers]: Ensure that requests for package come back as dataset - fixes several tests.
affected #: 1 file (-1 bytes)
--- a/ckan/controllers/api.py Thu Sep 29 17:21:54 2011 +0100
+++ b/ckan/controllers/api.py Fri Sep 30 15:55:21 2011 +0100
@@ -278,7 +278,8 @@
response_data = action(context, data_dict)
location = None
if "id" in data_dict:
- location = str('%s/%s' % (request.path, data_dict.get("id")))
+ location = str('%s/%s' % (request.path.replace('package', 'dataset'),
+ data_dict.get("id")))
return self._finish_ok(response_data,
resource_location=location)
except NotAuthorized:
http://bitbucket.org/okfn/ckan/changeset/0426089fef94/
changeset: 0426089fef94
branch: defect-1365-search-params
user: dread
date: 2011-09-30 16:55:47
summary: [branch] close
affected #: 0 files (-1 bytes)
http://bitbucket.org/okfn/ckan/changeset/a094002e1631/
changeset: a094002e1631
user: dread
date: 2011-09-30 17:00:10
summary: [merge] from defect-1365-search-params.
affected #: 14 files (-1 bytes)
--- a/ckan/config/routing.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/config/routing.py Fri Sep 30 16:00:10 2011 +0100
@@ -47,9 +47,9 @@
]
register_list_str = '|'.join(register_list)
- map.connect('/api/{ver:1|2}', controller='api', action='get_api')
+ map.connect('/api/{ver:1|2|3}', controller='api', action='get_api')
- map.connect('/api/{ver:1|2}/search/{register}', controller='api', action='search')
+ map.connect('/api/{ver:1|2|3}/search/{register}', controller='api', action='search')
map.connect('/api/{ver:1|2}/tag_counts', controller='api', action='tag_counts')
map.connect('/api/{ver:1|2}/rest', controller='api', action='index')
@@ -92,6 +92,8 @@
controller='api', action='delete',
requirements=dict(register=register_list_str),
conditions=dict(method=['DELETE']))
+ map.connect('/api/{ver:3}/action/{logic_function}', controller='api', action='action',
+ conditions=dict(method=['GET', 'POST']))
map.connect('/api/{ver:1|2}/qos/throughput/',
controller='api', action='throughput',
requirements=dict(register=register_list_str),
@@ -107,8 +109,8 @@
map.connect('/api/rest', controller='api', action='index')
- map.connect('/api/action/{logic_function}', controller='api', action='action')
-
+ map.connect('/api/action/{logic_function}', controller='api', action='action',
+ conditions=dict(method=['GET', 'POST']))
map.connect('/api/rest/{register}', controller='api', action='list',
requirements=dict(register=register_list_str),
conditions=dict(method=['GET'])
--- a/ckan/controllers/api.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/controllers/api.py Fri Sep 30 16:00:10 2011 +0100
@@ -7,7 +7,7 @@
from ckan.lib.helpers import json, date_str_to_datetime
import ckan.model as model
import ckan.rating
-from ckan.lib.search import query_for, QueryOptions, SearchIndexError, SearchError, DEFAULT_OPTIONS
+from ckan.lib.search import query_for, QueryOptions, SearchIndexError, SearchError, DEFAULT_OPTIONS, convert_legacy_parameters_to_solr
from ckan.plugins import PluginImplementations, IGroupController
from ckan.lib.munge import munge_title_to_name
from ckan.lib.navl.dictization_functions import DataError
@@ -278,7 +278,8 @@
response_data = action(context, data_dict)
location = None
if "id" in data_dict:
- location = str('%s/%s' % (request.path, data_dict.get("id")))
+ location = str('%s/%s' % (request.path.replace('package', 'dataset'),
+ data_dict.get("id")))
return self._finish_ok(response_data,
resource_location=location)
except NotAuthorized:
@@ -387,6 +388,7 @@
def search(self, ver=None, register=None):
log.debug('search %s params: %r' % (register, request.params))
+ ver = ver or '1' # i.e. default to v1
if register == 'revision':
since_time = None
if request.params.has_key('since_id'):
@@ -412,7 +414,7 @@
return self._finish_ok([rev.id for rev in revs])
elif register in ['dataset', 'package', 'resource']:
try:
- params = dict(self._get_search_params(request.params))
+ params = MultiDict(self._get_search_params(request.params))
except ValueError, e:
return self._finish_bad_request(
gettext('Could not read parameters: %r' % e))
@@ -451,7 +453,11 @@
query=params.get('q'), fields=query_fields, options=options
)
else:
- # for package searches we can pass parameters straight to Solr
+ # For package searches in API v3 and higher, we can pass
+ # parameters straight to Solr.
+ if ver in u'12':
+ # Otherwise, put all unrecognised ones into the q parameter
+ params = convert_legacy_parameters_to_solr(params)
query = query_for(model.Package)
results = query.run(params)
return self._finish_ok(results)
--- a/ckan/lib/search/__init__.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/lib/search/__init__.py Fri Sep 30 16:00:10 2011 +0100
@@ -7,7 +7,7 @@
from ckan.lib.dictization.model_dictize import package_to_api1
from common import SearchIndexError, SearchError, make_connection, is_available, DEFAULT_SOLR_URL
from index import PackageSearchIndex, NoopSearchIndex
-from query import TagSearchQuery, ResourceSearchQuery, PackageSearchQuery, QueryOptions
+from query import TagSearchQuery, ResourceSearchQuery, PackageSearchQuery, QueryOptions, convert_legacy_parameters_to_solr
log = logging.getLogger(__name__)
--- a/ckan/lib/search/common.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/lib/search/common.py Fri Sep 30 16:00:10 2011 +0100
@@ -4,6 +4,7 @@
class SearchIndexError(Exception): pass
class SearchError(Exception): pass
+class SearchQueryError(SearchError): pass
DEFAULT_SOLR_URL = 'http://127.0.0.1:8983/solr'
--- a/ckan/lib/search/query.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/lib/search/query.py Fri Sep 30 16:00:10 2011 +0100
@@ -1,9 +1,10 @@
-import json
from pylons import config
from paste.deploy.converters import asbool
+from paste.util.multidict import MultiDict
from ckan import model
from ckan.logic import get_action
-from common import make_connection, SearchError
+from ckan.lib.helpers import json
+from common import make_connection, SearchError, SearchQueryError
import logging
log = logging.getLogger(__name__)
@@ -11,7 +12,6 @@
VALID_SOLR_PARAMETERS = set([
'q', 'fl', 'fq', 'rows', 'sort', 'start', 'wt', 'qf',
- 'filter_by_downloadable', 'filter_by_openness',
'facet', 'facet.mincount', 'facet.limit', 'facet.field'
])
@@ -19,14 +19,63 @@
# and their relative weighting
QUERY_FIELDS = "name^4 title^4 tags^2 groups^2 text"
+def convert_legacy_parameters_to_solr(legacy_params):
+ '''API v1 and v2 allowed search params that the SOLR syntax does not
+ support, so use this function to convert those to SOLR syntax.
+ See tests for examples.
+
+ raises SearchQueryError on invalid params.
+ '''
+ options = QueryOptions(**legacy_params)
+ options.validate()
+ solr_params = legacy_params.copy()
+ solr_q_list = []
+ if solr_params.get('q'):
+ solr_q_list.append(solr_params['q'].replace('+', ' '))
+ non_solr_params = set(legacy_params.keys()) - VALID_SOLR_PARAMETERS
+ for search_key in non_solr_params:
+ value_obj = legacy_params[search_key]
+ value = str(value_obj).replace('+', ' ')
+ if search_key == 'all_fields':
+ if value:
+ solr_params['fl'] = '*'
+ elif search_key == 'offset':
+ solr_params['start'] = value
+ elif search_key == 'limit':
+ solr_params['rows'] = value
+ elif search_key == 'order_by':
+ solr_params['sort'] = '%s asc' % value
+ elif search_key == 'tags':
+ if isinstance(value_obj, list):
+ tag_list = value_obj
+ elif isinstance(value_obj, basestring):
+ tag_list = [value_obj]
+ else:
+ raise SearchQueryError('Was expecting either a string or JSON list for the tags parameter: %r' % value)
+ solr_q_list.extend(['tags:%s' % tag for tag in tag_list])
+ else:
+ if ' ' in value:
+ value = '"%s"' % value
+ solr_q_list.append('%s:%s' % (search_key, value))
+ del solr_params[search_key]
+ solr_params['q'] = ' '.join(solr_q_list)
+ if non_solr_params:
+ log.info('Converted legacy search params from %r to %r',
+ legacy_params, solr_params)
+ return solr_params
+
+
class QueryOptions(dict):
"""
Options specify aspects of the search query which are only tangentially related
to the query terms (such as limits, etc.).
+ NB This is used only by legacy package search and current resource & tag search.
+ Modern SOLR package search leaves this to SOLR syntax.
"""
- BOOLEAN_OPTIONS = ['filter_by_downloadable', 'filter_by_openness', 'all_fields']
+ BOOLEAN_OPTIONS = ['all_fields']
INTEGER_OPTIONS = ['offset', 'limit']
+ UNSUPPORTED_OPTIONS = ['filter_by_downloadable', 'filter_by_openness']
def __init__(self, **kwargs):
from ckan.lib.search import DEFAULT_OPTIONS
@@ -44,12 +93,14 @@
try:
value = asbool(value)
except ValueError:
- raise SearchError('Value for search option %r must be True or False (1 or 0) but received %r' % (key, value))
+ raise SearchQueryError('Value for search option %r must be True or False (1 or 0) but received %r' % (key, value))
elif key in self.INTEGER_OPTIONS:
try:
value = int(value)
except ValueError:
- raise SearchError('Value for search option %r must be an integer but received %r' % (key, value))
+ raise SearchQueryError('Value for search option %r must be an integer but received %r' % (key, value))
+ elif key in self.UNSUPPORTED_OPTIONS:
+ raise SearchQueryError('Search option %r is not supported' % key)
self[key] = value
def __getattr__(self, name):
@@ -173,12 +224,11 @@
return [r.get('id') for r in data.results]
def run(self, query):
- assert isinstance(query, dict)
-
+ assert isinstance(query, (dict, MultiDict))
# check that query keys are valid
if not set(query.keys()) <= VALID_SOLR_PARAMETERS:
invalid_params = [s for s in set(query.keys()) - VALID_SOLR_PARAMETERS]
- raise SearchError("Invalid search parameters: %s" % invalid_params)
+ raise SearchQueryError("Invalid search parameters: %s" % invalid_params)
# default query is to return all documents
q = query.get('q')
@@ -214,14 +264,6 @@
# return results as json encoded string
query['wt'] = query.get('wt', 'json')
- # check if filtering by downloadable or open license
- if int(query.get('filter_by_downloadable', 0)):
- query['fq'] += u" +res_url:[* TO *] " # not null resource URL
- if int(query.get('filter_by_openness', 0)):
- licenses = ["license_id:%s" % id for id in self.open_licenses]
- licenses = " OR ".join(licenses)
- query['fq'] += " +(%s) " % licenses
-
# query field weighting: disabled for now as solr 3.* is required for
# the 'edismax' query parser, our current Ubuntu version only has
# packages for 1.4
--- a/ckan/templates/layout_base.html Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/templates/layout_base.html Fri Sep 30 16:00:10 2011 +0100
@@ -101,7 +101,7 @@
<content><p>Master content template placeholder … please replace me.</p></content>
- </div>
+ </div><!-- /content --><div id="sidebar" class="span-7 last"><ul class="widget-list"><py:if test="defined('primary_sidebar_extras')">
--- a/ckan/tests/functional/api/base.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/tests/functional/api/base.py Fri Sep 30 16:00:10 2011 +0100
@@ -31,6 +31,7 @@
extra_environ = {}
api_version = None
+
ref_package_by = ''
ref_group_by = ''
@@ -68,14 +69,56 @@
base += '/' + self.api_version
return '%s%s' % (base, path)
- def package_offset(self, package_name=None):
- if package_name is None:
- # Package Register
- return self.offset('/rest/dataset')
- else:
- # Package Entity
- package_ref = self.package_ref_from_name(package_name)
- return self.offset('/rest/dataset/%s' % package_ref)
+ def assert_msg_represents_anna(self, msg):
+ assert 'annakarenina' in msg, msg
+ data = self.loads(msg)
+ self.assert_equal(data['name'], 'annakarenina')
+ self.assert_equal(data['license_id'], 'other-open')
+ assert '"license_id": "other-open"' in msg, str(msg)
+ assert 'russian' in msg, msg
+ assert 'tolstoy' in msg, msg
+ assert '"extras": {' in msg, msg
+ assert '"genre": "romantic novel"' in msg, msg
+ assert '"original media": "book"' in msg, msg
+ assert 'annakarenina.com/download' in msg, msg
+ assert '"plain text"' in msg, msg
+ assert '"Index of the novel"' in msg, msg
+ assert '"id": "%s"' % self.anna.id in msg, msg
+ expected = '"groups": ['
+ assert expected in msg, (expected, msg)
+ expected = self.group_ref_from_name('roger')
+ assert expected in msg, (expected, msg)
+ expected = self.group_ref_from_name('david')
+ assert expected in msg, (expected, msg)
+
+ # Todo: What is the deal with ckan_url? And should this use IDs rather than names?
+ assert 'ckan_url' in msg
+ assert '"ckan_url": "http://test.ckan.net/dataset/annakarenina"' in msg, msg
+
+ def assert_msg_represents_roger(self, msg):
+ assert 'roger' in msg, msg
+ data = self.loads(msg)
+ keys = set(data.keys())
+ expected_keys = set(['id', 'name', 'title', 'description', 'created',
+ 'state', 'revision_id', 'packages'])
+ missing_keys = expected_keys - keys
+ assert not missing_keys, missing_keys
+ assert_equal(data['name'], 'roger')
+ assert_equal(data['title'], 'Roger\'s books')
+ assert_equal(data['description'], 'Roger likes these books.')
+ assert_equal(data['state'], 'active')
+ assert_equal(data['packages'], [self._ref_package(self.anna)])
+
+ def assert_msg_represents_russian(self, msg):
+ data = self.loads(msg)
+ pkgs = set(data)
+ expected_pkgs = set([self.package_ref_from_name('annakarenina'),
+ self.package_ref_from_name('warandpeace')])
+ differences = expected_pkgs ^ pkgs
+ assert not differences, '%r != %r' % (pkgs, expected_pkgs)
+
+ def data_from_res(self, res):
+ return self.loads(res.body)
def package_ref_from_name(self, package_name):
package = self.get_package_by_name(unicode(package_name))
@@ -95,6 +138,41 @@
assert self.ref_package_by in ['id', 'name']
return getattr(package, self.ref_package_by)
+ def get_expected_api_version(self):
+ return self.api_version
+
+ def dumps(self, data):
+ return json.dumps(data)
+
+ def loads(self, chars):
+ try:
+ return json.loads(chars)
+ except ValueError, inst:
+ raise Exception, "Couldn't loads string '%s': %s" % (chars, inst)
+
+ def assert_json_response(self, res, expected_in_body=None):
+ content_type = res.header_dict['Content-Type']
+ assert 'application/json' in content_type, content_type
+ res_json = self.loads(res.body)
+ if expected_in_body:
+ assert expected_in_body in res_json or \
+ expected_in_body in str(res_json), \
+ 'Expected to find %r in JSON response %r' % \
+ (expected_in_body, res_json)
+
+class Api1and2TestCase(object):
+ ''' Utils for v1 and v2 API.
+ * RESTful URL utils
+ '''
+ def package_offset(self, package_name=None):
+ if package_name is None:
+ # Package Register
+ return self.offset('/rest/dataset')
+ else:
+ # Package Entity
+ package_ref = self.package_ref_from_name(package_name)
+ return self.offset('/rest/dataset/%s' % package_ref)
+
def group_offset(self, group_name=None):
if group_name is None:
# Group Register
@@ -197,81 +275,7 @@
def _list_group_refs(cls, groups):
return [getattr(p, cls.ref_group_by) for p in groups]
- def assert_msg_represents_anna(self, msg):
- assert 'annakarenina' in msg, msg
- data = self.loads(msg)
- self.assert_equal(data['name'], 'annakarenina')
- self.assert_equal(data['license_id'], 'other-open')
- assert '"license_id": "other-open"' in msg, str(msg)
- assert 'russian' in msg, msg
- assert 'tolstoy' in msg, msg
- assert '"extras": {' in msg, msg
- assert '"genre": "romantic novel"' in msg, msg
- assert '"original media": "book"' in msg, msg
- assert 'annakarenina.com/download' in msg, msg
- assert '"plain text"' in msg, msg
- assert '"Index of the novel"' in msg, msg
- assert '"id": "%s"' % self.anna.id in msg, msg
- expected = '"groups": ['
- assert expected in msg, (expected, msg)
- expected = self.group_ref_from_name('roger')
- assert expected in msg, (expected, msg)
- expected = self.group_ref_from_name('david')
- assert expected in msg, (expected, msg)
-
- # Todo: What is the deal with ckan_url? And should this use IDs rather than names?
- assert 'ckan_url' in msg
- assert '"ckan_url": "http://test.ckan.net/dataset/annakarenina"' in msg, msg
-
- def assert_msg_represents_roger(self, msg):
- assert 'roger' in msg, msg
- data = self.loads(msg)
- keys = set(data.keys())
- expected_keys = set(['id', 'name', 'title', 'description', 'created',
- 'state', 'revision_id', 'packages'])
- missing_keys = expected_keys - keys
- assert not missing_keys, missing_keys
- assert_equal(data['name'], 'roger')
- assert_equal(data['title'], 'Roger\'s books')
- assert_equal(data['description'], 'Roger likes these books.')
- assert_equal(data['state'], 'active')
- assert_equal(data['packages'], [self._ref_package(self.anna)])
-
- def assert_msg_represents_russian(self, msg):
- data = self.loads(msg)
- pkgs = set(data)
- expected_pkgs = set([self.package_ref_from_name('annakarenina'),
- self.package_ref_from_name('warandpeace')])
- differences = expected_pkgs ^ pkgs
- assert not differences, '%r != %r' % (pkgs, expected_pkgs)
-
- def data_from_res(self, res):
- return self.loads(res.body)
-
- def get_expected_api_version(self):
- return self.api_version
-
- def dumps(self, data):
- return json.dumps(data)
-
- def loads(self, chars):
- try:
- return json.loads(chars)
- except ValueError, inst:
- raise Exception, "Couldn't loads string '%s': %s" % (chars, inst)
-
- def assert_json_response(self, res, expected_in_body=None):
- content_type = res.header_dict['Content-Type']
- assert 'application/json' in content_type, content_type
- res_json = self.loads(res.body)
- if expected_in_body:
- assert expected_in_body in res_json or \
- expected_in_body in str(res_json), \
- 'Expected to find %r in JSON response %r' % \
- (expected_in_body, res_json)
-
-# Todo: Rename to Version1TestCase.
-class Api1TestCase(ApiTestCase):
+class Api1TestCase(Api1and2TestCase):
api_version = '1'
ref_package_by = 'name'
@@ -283,7 +287,7 @@
assert '"download_url": "http://www.annakarenina.com/download/x=1&y=2"' in msg, msg
-class Api2TestCase(ApiTestCase):
+class Api2TestCase(Api1and2TestCase):
api_version = '2'
ref_package_by = 'id'
@@ -295,6 +299,17 @@
assert 'download_url' not in msg, msg
+class Api3TestCase(ApiTestCase):
+
+ api_version = '3'
+ ref_package_by = 'name'
+ ref_group_by = 'name'
+ ref_tag_by = 'name'
+
+ def assert_msg_represents_anna(self, msg):
+ super(Api2TestCase, self).assert_msg_represents_anna(msg)
+ assert 'download_url' not in msg, msg
+
class ApiUnversionedTestCase(Api1TestCase):
api_version = ''
--- a/ckan/tests/functional/api/test_misc.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/tests/functional/api/test_misc.py Fri Sep 30 16:00:10 2011 +0100
@@ -42,5 +42,5 @@
class TestQosApi1(Api1TestCase, QosApiTestCase): pass
class TestMiscApi2(Api2TestCase, MiscApiTestCase): pass
class TestQosApi2(Api2TestCase, QosApiTestCase): pass
-class TestMiscApiUnversioned(MiscApiTestCase, ApiUnversionedTestCase): pass
+class TestMiscApiUnversioned(ApiUnversionedTestCase, MiscApiTestCase): pass
class TestQosApiUnversioned(ApiUnversionedTestCase, QosApiTestCase): pass
--- a/ckan/tests/functional/api/test_package_search.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/tests/functional/api/test_package_search.py Fri Sep 30 16:00:10 2011 +0100
@@ -1,4 +1,5 @@
from nose.tools import assert_raises
+from nose.plugins.skip import SkipTest
from ckan import plugins
import ckan.lib.search as search
@@ -38,10 +39,6 @@
for expected_package_name in expected_package_names]
assert_equal(set(res_dict['results']), set(expected_pkgs))
- def package_ref_from_name(self, package_name):
- package = self.get_package_by_name(package_name)
- return self.ref_package(package)
-
def test_00_read_search_params(self):
def check(request_params, expected_params):
params = ApiController._get_search_params(request_params)
@@ -49,8 +46,6 @@
# uri parameters
check(UnicodeMultiDict({'q': '', 'ref': 'boris'}),
{"q": "", "ref": "boris"})
- check(UnicodeMultiDict({}),
- {})
# uri json
check(UnicodeMultiDict({'qjson': '{"q": "", "ref": "boris"}'}),
{"q": "", "ref": "boris"})
@@ -75,7 +70,6 @@
offset = self.base_url + '?q=%s' % self.package_fixture_data['name']
res = self.app.get(offset, status=200)
res_dict = self.data_from_res(res)
- print res_dict
self.assert_results(res_dict, ['testpkg'])
assert res_dict['count'] == 1, res_dict['count']
@@ -131,6 +125,225 @@
self.assert_results(res_dict, [u'annakarenina'])
assert res_dict['count'] == 1, res_dict['count']
+ def test_08_uri_qjson_malformed(self):
+ offset = self.base_url + '?qjson="q":""' # user forgot the curly braces
+ res = self.app.get(offset, status=400)
+ self.assert_json_response(res, 'Bad request - Could not read parameters')
+
+ def test_09_just_tags(self):
+ offset = self.base_url + '?q=tags:russian'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+
+ def test_10_multiple_tags(self):
+ offset = self.base_url + '?q=tags:tolstoy tags:russian'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_12_all_packages_qjson(self):
+ query = {'q': ''}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 3)
+
+ def test_12_all_packages_q(self):
+ offset = self.base_url + '?q=""'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 3)
+
+ def test_12_all_packages_no_q(self):
+ offset = self.base_url
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 3)
+
+ def test_12_filter_by_openness(self):
+ offset = self.base_url + '?filter_by_openness=1'
+ res = self.app.get(offset, status=400) # feature dropped in #1360
+ assert "'filter_by_openness'" in res.body, res.body
+
+ def test_12_filter_by_downloadable(self):
+ offset = self.base_url + '?filter_by_downloadable=1'
+ res = self.app.get(offset, status=400) # feature dropped in #1360
+ assert "'filter_by_downloadable'" in res.body, res.body
+
+
+class LegacyOptionsTestCase(ApiTestCase, ControllerTestCase):
+ '''Here are tests with URIs in the syntax they were in
+ for API v1 and v2.'''
+ @classmethod
+ def setup_class(self):
+ setup_test_search_index()
+ CreateTestData.create()
+ self.package_fixture_data = {
+ 'name' : u'testpkg',
+ 'title': 'Some Title',
+ 'url': u'http://blahblahblah.mydomain',
+ 'resources': [{u'url':u'http://blahblahblah.mydomain',
+ u'format':u'', u'description':''}],
+ 'tags': ['russion', 'novel'],
+ 'license_id': u'gpl-3.0',
+ 'extras': {'national_statistic':'yes',
+ 'geographic_coverage':'England, Wales'},
+ }
+ CreateTestData.create_arbitrary(self.package_fixture_data)
+ self.base_url = self.offset('/search/dataset')
+
+ @classmethod
+ def teardown_class(cls):
+ model.repo.rebuild_db()
+ search.clear()
+
+ def test_07_uri_qjson_tags(self):
+ query = {'q': '', 'tags':['tolstoy']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_tags_multiple(self):
+ query = {'q': '', 'tags':['tolstoy', 'russian']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ print offset
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_tags_reverse(self):
+ query = {'q': '', 'tags':['russian']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina', u'warandpeace'])
+ assert res_dict['count'] == 2, res_dict
+
+ def test_07_uri_qjson_extras(self):
+ # TODO: solr is not currently set up to allow partial matches
+ # and extras are not saved as multivalued so this
+ # test will fail. Make extras multivalued or remove?
+ raise SkipTest()
+
+ query = {"geographic_coverage":"England"}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_extras_2(self):
+ query = {"national_statistic":"yes"}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_08_all_fields(self):
+ rating = model.Rating(user_ip_address=u'123.1.2.3',
+ package=self.anna,
+ rating=3.0)
+ model.Session.add(rating)
+ model.repo.commit_and_remove()
+
+ query = {'q': 'russian', 'all_fields': 1}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ for rec in res_dict['results']:
+ if rec['name'] == 'annakarenina':
+ anna_rec = rec
+ break
+ assert anna_rec['name'] == 'annakarenina', res_dict['results']
+ assert anna_rec['title'] == 'A Novel By Tolstoy', anna_rec['title']
+ assert anna_rec['license_id'] == u'other-open', anna_rec['license_id']
+ assert len(anna_rec['tags']) == 2, anna_rec['tags']
+ for expected_tag in ['russian', 'tolstoy']:
+ assert expected_tag in anna_rec['tags']
+
+ # try alternative syntax
+ offset = self.base_url + '?q=russian&all_fields=1'
+ res2 = self.app.get(offset, status=200)
+ assert_equal(res2.body, res.body)
+
+ def test_08_all_fields_syntax_error(self):
+ offset = self.base_url + '?all_fields=should_be_boolean' # invalid all_fields value
+ res = self.app.get(offset, status=400)
+ assert('boolean' in res.body)
+ assert('all_fields' in res.body)
+ self.assert_json_response(res, 'boolean')
+
+ def test_09_just_tags(self):
+ offset = self.base_url + '?tags=tolstoy'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_10_multiple_tags_with_plus(self):
+ offset = self.base_url + '?tags=tolstoy+russian&all_fields=1'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_10_multiple_tags_with_ampersand(self):
+ offset = self.base_url + '?tags=tolstoy&tags=russian&all_fields=1'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_10_many_tags_with_ampersand(self):
+ offset = self.base_url + '?tags=tolstoy&tags=russian&tags=tolstoy'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_11_pagination_limit(self):
+ offset = self.base_url + '?all_fields=1&q=tags:russian&limit=1&order_by=name'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ assert len(res_dict['results']) == 1, res_dict
+ assert res_dict['results'][0]['name'] == 'annakarenina', res_dict['results'][0]['name']
+
+ def test_11_pagination_offset_limit(self):
+ offset = self.base_url + '?all_fields=1&q=tags:russian&offset=1&limit=1&order_by=name'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ assert len(res_dict['results']) == 1, res_dict
+ assert res_dict['results'][0]['name'] == 'warandpeace', res_dict['results'][0]['name']
+
+ def test_11_pagination_syntax_error(self):
+ offset = self.base_url + '?all_fields=1&q="tags:russian"&start=should_be_integer&rows=1&order_by=name' # invalid offset value
+ res = self.app.get(offset, status=400)
+ print res.body
+ assert('should_be_integer' in res.body)
+
+ def test_13_just_groups(self):
+ offset = self.base_url + '?groups=roger'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+class TestPackageSearchApi1(Api1TestCase, PackageSearchApiTestCase,
+ LegacyOptionsTestCase): pass
+class TestPackageSearchApi2(Api2TestCase, PackageSearchApiTestCase,
+ LegacyOptionsTestCase): pass
+class TestPackageSearchApi3(Api3TestCase, PackageSearchApiTestCase):
+ '''Here are tests with URIs in specifically SOLR syntax.'''
def test_07_uri_qjson_tags(self):
query = {'q': 'tags:tolstoy'}
json_query = self.dumps(query)
@@ -159,21 +372,6 @@
self.assert_results(res_dict, [u'annakarenina', u'warandpeace'])
assert res_dict['count'] == 2, res_dict
- def test_07_uri_qjson_extras(self):
- # TODO: solr is not currently set up to allow partial matches
- # and extras are not saved as multivalued so this
- # test will fail. Make extras multivalued or remove?
- from ckan.tests import SkipTest
- raise SkipTest
-
- query = {"geographic_coverage":"England"}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_results(res_dict, ['testpkg'])
- assert res_dict['count'] == 1, res_dict
-
def test_07_uri_qjson_extras_2(self):
query = {'q': "national_statistic:yes"}
json_query = self.dumps(query)
@@ -183,18 +381,7 @@
self.assert_results(res_dict, ['testpkg'])
assert res_dict['count'] == 1, res_dict
- def test_08_uri_qjson_malformed(self):
- offset = self.base_url + '?qjson="q":""' # user forgot the curly braces
- res = self.app.get(offset, status=400)
- self.assert_json_response(res, 'Bad request - Could not read parameters')
-
def test_08_all_fields(self):
- rating = model.Rating(user_ip_address=u'123.1.2.3',
- package=self.anna,
- rating=3.0)
- model.Session.add(rating)
- model.repo.commit_and_remove()
-
query = {'q': 'russian', 'fl': '*'}
json_query = self.dumps(query)
offset = self.base_url + '?qjson=%s' % json_query
@@ -212,62 +399,17 @@
for expected_tag in ['russian', 'tolstoy']:
assert expected_tag in anna_rec['tags']
- # TODO: these values are not being passed to Solr
- # assert anna_rec['ratings_average'] == 3.0, anna_rec['ratings_average']
- # assert anna_rec['ratings_count'] == 1, anna_rec['ratings_count']
-
# try alternative syntax
offset = self.base_url + '?q=russian&fl=*'
res2 = self.app.get(offset, status=200)
assert_equal(res2.body, res.body)
- def test_08_all_fields_syntax_error(self):
- offset = self.base_url + '?all_fields=should_be_boolean' # invalid all_fields value
- res = self.app.get(offset, status=400)
- assert('all_fields' in res.body)
-
def test_09_just_tags(self):
offset = self.base_url + '?q=tags:russian&fl=*'
res = self.app.get(offset, status=200)
res_dict = self.data_from_res(res)
assert res_dict['count'] == 2, res_dict
- def test_10_multiple_tags(self):
- offset = self.base_url + '?q=tags:tolstoy tags:russian&fl=*'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
- def test_10_multiple_tags_with_plus(self):
- # TODO: this syntax doesn't work with Solr search, update documentation
- from nose import SkipTest
- raise SkipTest
-
- offset = self.base_url + '?tags=tolstoy+russian&all_fields=1'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
- def test_10_multiple_tags_with_ampersand(self):
- # TODO: this syntax doesn't work with Solr search, update documentation
- from nose import SkipTest
- raise SkipTest
-
- offset = self.base_url + '?tags=tolstoy&tags=russian&all_fields=1'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
- def test_10_many_tags_with_ampersand(self):
- # TODO: this syntax doesn't work with Solr search, update documentation
- from nose import SkipTest
- raise SkipTest
-
- offset = self.base_url + '?tags=tolstoy&tags=russian&tags=tolstoy'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
def test_11_pagination_limit(self):
offset = self.base_url + '?fl=*&q=tags:russian&rows=1&sort=name asc'
res = self.app.get(offset, status=200)
@@ -285,30 +427,15 @@
assert res_dict['results'][0]['name'] == 'warandpeace', res_dict['results'][0]['name']
def test_11_pagination_syntax_error(self):
- offset = self.base_url + '?fl=*&q="tags:russian"&start=should_be_integer&rows=1&sort=name' # invalid offset value
+ offset = self.base_url + '?fl=*&q=tags:russian&start=should_be_integer&rows=1&sort=name asc' # invalid offset value
res = self.app.get(offset, status=400)
print res.body
assert('should_be_integer' in res.body)
- def test_12_all_packages_qjson(self):
- query = {'q': ''}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert_equal(res_dict['count'], 3)
-
- def test_12_all_packages_q(self):
- offset = self.base_url + '?q=""'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert_equal(res_dict['count'], 3)
-
- def test_12_all_packages_no_q(self):
- offset = self.base_url
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert_equal(res_dict['count'], 3)
+ def test_12_v1_or_v2_syntax(self):
+ offset = self.base_url + '?all_fields=1'
+ res = self.app.get(offset, status=400)
+ assert("Invalid search parameters: ['all_fields']" in res.body), res.body
def test_13_just_groups(self):
offset = self.base_url + '?q=groups:roger'
@@ -316,9 +443,10 @@
res_dict = self.data_from_res(res)
assert res_dict['count'] == 1, res_dict
-class TestPackageSearchApi1(Api1TestCase, PackageSearchApiTestCase): pass
-class TestPackageSearchApi2(Api2TestCase, PackageSearchApiTestCase): pass
-class TestPackageSearchApiUnversioned(PackageSearchApiTestCase, ApiUnversionedTestCase): pass
+class TestPackageSearchApiUnversioned(PackageSearchApiTestCase,
+ ApiUnversionedTestCase,
+ LegacyOptionsTestCase): pass
+
--- a/ckan/tests/functional/api/test_resource_search.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/tests/functional/api/test_resource_search.py Fri Sep 30 16:00:10 2011 +0100
@@ -79,7 +79,7 @@
assert 'package_id' in result.body, result.body
-class TestResourceSearchApi1(ResourceSearchApiTestCase, Api1TestCase): pass
+class TestResourceSearchApi1(Api1TestCase, ResourceSearchApiTestCase): pass
class TestResourceSearchApi2(Api2TestCase, ResourceSearchApiTestCase): pass
class TestResourceSearchApiUnversioned(ApiUnversionedTestCase, ResourceSearchApiTestCase):
pass
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckan/tests/functional/test_search.py Fri Sep 30 16:00:10 2011 +0100
@@ -0,0 +1,68 @@
+# These only test that the controller is passing on queries correctly
+# to the search library. The search library is tested in:
+# ckan/tests/lib/test_solr_package_search.py
+
+import re
+from nose.tools import assert_equal
+
+from ckan.tests import (TestController, CreateTestData,
+ setup_test_search_index, html_check)
+from ckan import model
+import ckan.lib.search as search
+
+class TestSearch(TestController, html_check.HtmlCheckMethods):
+ # 'penguin' is in all test search packages
+ q_all = u'penguin'
+
+ @classmethod
+ def setup_class(cls):
+ model.Session.remove()
+ setup_test_search_index()
+ CreateTestData.create_search_test_data()
+ cls.count_re = re.compile('<strong>(\d)</strong> datasets found')
+
+ @classmethod
+ def teardown_class(cls):
+ model.repo.rebuild_db()
+ search.clear()
+
+ def _pkg_names(self, result):
+ return ' '.join(result['results'])
+
+ def _check_results(self, res, expected_count, expected_package_names=[]):
+ '''Takes a search result web page and determines whether the
+ search results displayed match the expected count and names
+ of packages.'''
+ # get count
+ content = self.named_div('content', res)
+ count_match = self.count_re.search(content)
+ assert count_match
+ assert_equal(len(count_match.groups()), 1)
+ count = int(count_match.groups()[0])
+ assert_equal(count, expected_count)
+
+ # check package names
+ if isinstance(expected_package_names, basestring):
+ expected_package_names = [expected_package_names]
+ for expected_name in expected_package_names:
+ expected_html = '<a href="/dataset/%s">' % expected_name
+ assert expected_html in res.body, \
+ 'Could not find package name %r in the results page'
+
+ def test_1_all_records(self):
+ res = self.app.get('/dataset?q')
+ result = self._check_results(res, 6, 'gils')
+
+ def test_1_name(self):
+ # exact name
+ res = self.app.get('/dataset?q=gils')
+ result = self._check_results(res, 1, 'gils')
+
+ def test_2_title(self):
+ # exact title, one word
+ res = self.app.get('/dataset?q=Opengov.se')
+ result = self._check_results(res, 1, 'se-opengov')
+
+ # multiple words
+ res = self.app.get('/dataset?q=Government%20Expenditure')
+ result = self._check_results(res, 1, 'uk-government-expenditure')
--- a/ckan/tests/lib/test_solr_package_search.py Wed Sep 28 16:29:57 2011 +0100
+++ b/ckan/tests/lib/test_solr_package_search.py Fri Sep 30 16:00:10 2011 +0100
@@ -1,8 +1,29 @@
+from nose.tools import assert_equal, assert_raises
+
from ckan.tests import TestController, CreateTestData, setup_test_search_index
from ckan import model
import ckan.lib.search as search
+class TestQuery:
+ def test_1_convert_legacy_params_to_solr(self):
+ convert = search.convert_legacy_parameters_to_solr
+ assert_equal(convert({'title': 'bob'}), {'q': 'title:bob'})
+ assert_equal(convert({'title': 'bob', 'fl': 'name'}),
+ {'q': 'title:bob', 'fl': 'name'})
+ assert_equal(convert({'title': 'bob perkins'}), {'q': 'title:"bob perkins"'})
+ assert_equal(convert({'q': 'high+wages'}), {'q': 'high wages'})
+ assert_equal(convert({'q': 'high+wages summary'}), {'q': 'high wages summary'})
+ assert_equal(convert({'title': 'high+wages'}), {'q': 'title:"high wages"'})
+ assert_equal(convert({'title': 'bob', 'all_fields': 1}), {'q': 'title:bob', 'fl': '*'})
+ assert_raises(search.SearchError, convert, {'title': 'bob', 'all_fields': 'non-boolean'})
+ assert_equal(convert({'q': 'bob', 'order_by': 'name'}), {'q': 'bob', 'sort':'name asc'})
+ assert_equal(convert({'q': 'bob', 'offset': '0', 'limit': '10'}), {'q': 'bob', 'start':'0', 'rows':'10'})
+ assert_equal(convert({'tags': ['russian', 'tolstoy']}), {'q': 'tags:russian tags:tolstoy'})
+ assert_equal(convert({'tags': ['tolstoy']}), {'q': 'tags:tolstoy'})
+ assert_equal(convert({'tags': 'tolstoy'}), {'q': 'tags:tolstoy'})
+ assert_raises(search.SearchError, convert, {'tags': {'tolstoy':1}})
+
class TestSearch(TestController):
# 'penguin' is in all test search packages
q_all = u'penguin'
@@ -253,11 +274,9 @@
model.repo.rebuild_db()
search.clear()
- def _check_search_results(self, terms, expected_count, expected_packages=[], only_open=False, only_downloadable=False):
+ def _check_search_results(self, terms, expected_count, expected_packages=[]):
query = {
'q': unicode(terms),
- 'filter_by_openness': only_open,
- 'filter_by_downloadable': only_downloadable
}
result = search.query_for(model.Package).run(query)
pkgs = result['results']
@@ -277,9 +296,6 @@
self._check_search_results('groups:david', 2)
self._check_search_results('groups:roger', 1)
self._check_search_results('groups:lenny', 0)
- self._check_search_results('annakarenina', 1, ['annakarenina'], True, False)
- self._check_search_results('annakarenina', 1, ['annakarenina'], False, True)
- self._check_search_results('annakarenina', 1, ['annakarenina'], True, True)
class TestGeographicCoverage(TestController):
--- a/doc/action_api.rst Wed Sep 28 16:29:57 2011 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,241 +0,0 @@
-.. index:: API
-
-===========================
-Reference: CKAN Action API
-===========================
-
-.. toctree::
- :hidden:
- :maxdepth: 1
-
-.. warning:: The Action API is still experimental and subject to change of URI locations, formats, parameters and results.
-
-Overview
---------
-
-The Action API is a powerful RPC-style way of accessing CKAN data. Its intention is to have access to all the core logic in ckan. It calls exactly the same functions that are used internally which all the other CKAN interfaces (Web interface / Model API) go through. Therefore it provides the full gamut of read and write operations, with all possible parameters.
-
-A client supplies parameters to the Action API via a JSON dictionary of a POST request, and returns results, help information and any error diagnostics in a JSON dictionary too. This is a departure from the CKAN API versions 1 and 2, which being RESTful required all the request parameters to be part of the URL.
-
-Requests
---------
-
-URL
-===
-
-The basic URL for the Action API is::
-
- /api/action/{logic_action}
-
-Examples::
-
- /api/action/package_list
- /api/action/package_show
- /api/action/user_create
-
-Actions
-=======
-
-get.py:
-
-====================================== ===========================
-Logic Action Parameter keys
-====================================== ===========================
-site_read (none)
-package_list (none)
-current_package_list_with_resources limit
-revision_list (none)
-package_revision_list id
-group_list all_fields
-group_list_authz (none)
-group_list_available (none)
-group_revision_list id
-licence_list (none)
-tag_list q, all_fields, limit, offset, return_objects
-user_list q, order_by
-package_relationships_list id, id2, rel
-package_show id
-revision_show id
-group_show id
-tag_show id
-user_show id
-package_show_rest id
-group_show_rest id
-tag_show_rest id
-package_autocomplete q
-tag_autocomplete q, limit
-format_autocomplete q, limit
-user_autocomplete q, limit
-package_search q, fields, facet_by, limit, offset, filter_by_openness, filter_by_downloadable
-====================================== ===========================
-
-new.py:
-
-====================================== ===========================
-Logic Action Parameter keys
-====================================== ===========================
-package_create (package keys)
-package_create_validate (package keys)
-resource_create (resource keys)
-package_relationship_create id, id2, rel, comment
-group_create (group keys)
-rating_create package, rating
-user_create (user keys)
-package_create_rest (package keys)
-group_create_rest (group keys)
-====================================== ===========================
-
-update.py:
-
-====================================== ===========================
-Logic Action Parameter keys
-====================================== ===========================
-make_latest_pending_package_active id
-resource_update (resource keys)
-package_update (package keys)
-package_update_validate (package keys)
-package_relationship_update id, id2, rel, comment
-group_update (group keys)
-user_update (user keys), reset_key
-package_update_rest (package keys)
-group_update_rest (group keys)
-====================================== ===========================
-
-delete.py:
-
-====================================== ===========================
-Logic Action Parameter keys
-====================================== ===========================
-package_delete id
-package_relationship_delete id, id2, rel
-group_delete id
-====================================== ===========================
-
-In case of doubt, refer to the code of the logic actions, which is found in the CKAN source in the ckan/logic/action directory.
-
-Object dictionaries
-===================
-
-Package:
-
-======================== ====================================== =============
-key example value notes
-======================== ====================================== =============
-id "fd788e57-dce4-481c-832d-497235bf9f78" (Read-only) unique identifier
-name "uk-spending" Unique identifier. Should be human readable
-title "UK Spending" Human readable title of the dataset
-url "http://gov.uk/spend-downloads.html" Home page for the data
-version "1.0" Version associated with the data. String format.
-author "UK Treasury" Name of person responsible for the data
-author_email "contact at treasury.gov.uk" Email address for the person in the 'author' field
-maintainer null Name of another person responsible for the data
-maintainer_email null Email address for the person in the 'maintainer' field
-notes "### About\\r\\n\\r\\nUpdated 1997." Other human readable info about the dataset. Markdown format.
-license_id "cc-by" ID of the license this dataset is released under. You can then look up the license ID to get the title.
-extras []
-tags ["government-spending"] List of tags associated with this dataset.
-groups ["spending", "country-uk"] List of groups this dataset is a member of.
-relationships_as_subject [] List of relationships (edit this only using relationship specific command). The 'type' of the relationship is described in terms of this package being the subject and the related package being the object.
-state active May be ``deleted`` or other custom states like ``pending``.
-revision_id "f645243a-7334-44e2-b87c-64231700a9a6" (Read-only) ID of the last revision for the core package object was (doesn't include tags, groups, extra fields, relationships).
-revision_timestamp "2010-12-21T15:26:17.345502" (Read-only) Time and date when the last revision for the core package object was (doesn't include tags, groups, extra fields, relationships). ISO format. UTC timezone assumed.
-======================== ====================================== =============
-
-Package Extra:
-
-======================== ====================================== =============
-key example value notes
-======================== ====================================== =============
-id "c10fb749-ad46-4ba2-839a-41e8e2560687" (Read-only)
-key "number_of_links"
-value "10000"
-package_id "349259a8-cbff-4610-8089-2c80b34e27c5" (Read-only) Edit package extras with package_update
-state "active" (Read-only) Edit package extras with package_update
-revision_timestamp "2010-09-01T08:56:53.696551" (Read-only)
-revision_id "233d0c19-fcdc-44b9-9afe-25e2aa9d0a5f" (Read-only)
-======================== ====================================== =============
-
-
-Resource:
-
-======================== ====================================== =============
-key example value notes
-======================== ====================================== =============
-id "888d00e9-6ee5-49ca-9abb-6f216e646345" (Read-only)
-url "http://gov.uk/spend-july-2009.csv" Download URL of the data
-description ""
-format "XLS" Format of the data
-hash null Hash of the data e.g. SHA1
-state "active"
-position 0 (Read-only) This is set by the order of resources are given in the list when creating/updating the package.
-resource_group_id "49ddadb0-dd80-9eff-26e9-81c5a466cf6e" (Read-only)
-revision_id "188ac88b-1573-48bf-9ea6-d3c503db5816" (Read-only)
-revision_timestamp "2011-07-08T14:48:38.967741" (Read-only)
-======================== ====================================== =============
-
-Tag:
-
-======================== ====================================== =============
-key example value notes
-======================== ====================================== =============
-id "b10871ea-b4ae-4e2e-bec9-a8d8ff357754" (Read-only)
-name "country-uk" (Read-only) Add/remove tags from a package or group using update_package or update_group
-state "active" (Read-only) Add/remove tags from a package or group using update_package or update_group
-revision_timestamp "2009-08-08T12:46:40.920443" (Read-only)
-======================== ====================================== =============
-
-Parameters
-==========
-
-Requests must be a POST, including parameters in a JSON dictionary. If there are no parameters required, then an empty dictionary is still required (or you get a 400 error).
-
-Examples::
-
- curl http://test.ckan.net/api/action/package_list -d '{}'
- curl http://test.ckan.net/api/action/package_show -d '{"id": "fd788e57-dce4-481c-832d-497235bf9f78"}'
-
-Authorization Header
-====================
-
-Authorization is carried out the same way as the existing API, supplying the user's API key in the "Authorization" header.
-
-Depending on the settings of the instance, you may not need to identify yourself for simple read operations. (This is the case for thedatahub.org and is assumed for the examples below.)
-
-JSONP
-=====
-
-TBC
-
-Responses
-=========
-
-The response is wholly contained in the form of a JSON dictionary. Here is the basic format of a successful request::
-
- {"help": "Creates a package", "success": true, "result": ...}
-
-And here is one that incurred an error::
-
- {"help": "Creates a package", "success": false, "error": {"message": "Access denied", "__type": "Authorization Error"}}
-
-Where:
-
-* ``help`` is the 'doc string' (or ``null``)
-* ``success`` is ``true`` or ``false`` depending on whether the request was successful. The response is always status 200, so it is important to check this value.
-* ``result`` is the main payload that results from a successful request. This might be a list of the domain object names or a dictionary with the particular domain object.
-* ``error`` is supplied if the request was not successful and provides a message and __type. See the section on errors.
-
-Errors
-======
-
-The message types include:
- * Authorization Error - an API key is required for this operation, and the corresponding user needs the correct credentials
- * Validation Error - the object supplied does not meet with the standards described in the schema.
- * (TBC) JSON Error - the request could not be parsed / decoded as JSON format, according to the Content-Type (default is ``application/x-www-form-urlencoded;utf-8``).
-
-Examples
-========
-
-::
-
- $ curl http://ckan.net/api/action/package_show -d '{"id": "fd788e57-dce4-481c-832d-497235bf9f78"}'
- {"help": null, "success": true, "result": {"maintainer": null, "name": "uk-quango-data", "relationships_as_subject": [], "author": null, "url": "http://www.guardian.co.uk/news/datablog/2009/jul/07/public-finance-regulators", "relationships_as_object": [], "notes": "### About\r\n\r\nDid you know there are nearly 1,200 unelected bodies with power over our lives? This is the full list, complete with number of staff and how much they cost. As a spreadsheet\r\n\r\n### Openness\r\n\r\nNo licensing information found.", "title": "Every Quango in Britain", "maintainer_email": null, "revision_timestamp": "2010-12-21T15:26:17.345502", "author_email": null, "state": "active", "version": null, "groups": [], "license_id": "notspecified", "revision_id": "f645243a-7334-44e2-b87c-64231700a9a6", "tags": [{"revision_timestamp": "2009-08-08T12:46:40.920443", "state": "active", "id": "b10871ea-b4ae-4e2e-bec9-a8d8ff357754", "name": "country-uk"}, {"revision_timestamp": "2009-08-08T12:46:40.920443", "state": "active", "id": "ed783bc3-c0a1-49f6-b861-fd9adbc1006b", "name": "quango"}], "id": "fd788e57-dce4-481c-832d-497235bf9f78", "resources": [{"resource_group_id": "49ddadb0-dd80-9eff-26e9-81c5a466cf6e", "hash": null, "description": "", "format": "", "url": "http://spreadsheets.google.com/ccc?key=tm4Dxoo0QtDrEOEC1FAJuUg", "revision_timestamp": "2011-07-08T14:48:38.967741", "state": "active", "position": 0, "revision_id": "188ac88b-1573-48bf-9ea6-d3c503db5816", "id": "888d00e9-6ee5-49ca-9abb-6f216e646345"}], "extras": []}}
\ No newline at end of file
--- a/doc/api.rst Wed Sep 28 16:29:57 2011 +0100
+++ b/doc/api.rst Fri Sep 30 16:00:10 2011 +0100
@@ -59,6 +59,12 @@
The only exception for this is for Tag objects. Since Tag names are immutable, they are always referred to with their Name.
+Version 3
+~~~~~~~~~
+
+This version is in beta. To details of trying it out, see :doc:`apiv3`.
+
+
API Details - Versions 1 & 2
----------------------------
@@ -85,7 +91,6 @@
* `Model API`_
* `Search API`_
* `Util API`_
-* `Action API`_
The resources, methods, and data formats of each are described below.
@@ -329,7 +334,7 @@
| | | | parameter as a more flexible |
| | | | alternative in GET requests. |
+-----------------------+---------------+----------------------------------+----------------------------------+
-|title, | Search-String || title=uk&tags=health+census | Search in a particular a field. |
+|title, | Search-String || title=uk&tags=health | Search in a particular a field. |
|tags, notes, groups, | || department=environment | |
|author, maintainer, | | | |
|update_frequency, or | | | |
@@ -349,12 +354,9 @@
| | | | (0) or the full dataset record |
| | | | (1). |
+-----------------------+---------------+----------------------------------+----------------------------------+
-| filter_by_openness | 0 (default) | filter_by_openness=1 | Filters results by ones which are|
-| | or 1 | | open. |
-+-----------------------+---------------+----------------------------------+----------------------------------+
-|filter_by_downloadable | 0 (default) | filter_by_downloadable=1 | Filters results by ones which |
-| | or 1 | | have at least one resource URL. |
-+-----------------------+---------------+----------------------------------+----------------------------------+
+
+.. Note: filter_by_openness and filter_by_downloadable were dropped from CKAN version 1.5 onwards.
+
**Resource Parameters**
@@ -554,4 +556,4 @@
Action API
~~~~~~~~~~
-See: :doc:`action_api`
\ No newline at end of file
+See:
\ No newline at end of file
Repository URL: https://bitbucket.org/okfn/ckan/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list