[ckan-changes] commit/ckan: dread: [lib]: Basic translator of search param added in. Tests getting there. Docs for v3 better laid out. Routing adjusted for new api v3.
Bitbucket
commits-noreply at bitbucket.org
Wed Sep 28 18:06:49 UTC 2011
1 new changeset in ckan:
http://bitbucket.org/okfn/ckan/changeset/fa5f03352e5f/
changeset: fa5f03352e5f
branch: defect-1365-search-params
user: dread
date: 2011-09-28 20:06:06
summary: [lib]: Basic translator of search param added in. Tests getting there. Docs for v3 better laid out. Routing adjusted for new api v3.
affected #: 10 files (-1 bytes)
--- a/ckan/config/routing.py Wed Sep 28 11:45:57 2011 +0100
+++ b/ckan/config/routing.py Wed Sep 28 19:06:06 2011 +0100
@@ -47,9 +47,9 @@
]
register_list_str = '|'.join(register_list)
- map.connect('/api/{ver:1|2}', controller='api', action='get_api')
+ map.connect('/api/{ver:1|2|3}', controller='api', action='get_api')
- map.connect('/api/{ver:1|2}/search/{register}', controller='api', action='search')
+ map.connect('/api/{ver:1|2|3}/search/{register}', controller='api', action='search')
map.connect('/api/{ver:1|2}/tag_counts', controller='api', action='tag_counts')
map.connect('/api/{ver:1|2}/rest', controller='api', action='index')
@@ -92,6 +92,8 @@
controller='api', action='delete',
requirements=dict(register=register_list_str),
conditions=dict(method=['DELETE']))
+ map.connect('/api/{ver:3}/action/{logic_function}', controller='api', action='action',
+ conditions=dict(method=['GET']))
map.connect('/api/{ver:1|2}/qos/throughput/',
controller='api', action='throughput',
requirements=dict(register=register_list_str),
@@ -107,8 +109,8 @@
map.connect('/api/rest', controller='api', action='index')
- map.connect('/api/action/{logic_function}', controller='api', action='action')
-
+ map.connect('/api/action/{logic_function}', controller='api', action='action',
+ conditions=dict(method=['GET']))
map.connect('/api/rest/{register}', controller='api', action='list',
requirements=dict(register=register_list_str),
conditions=dict(method=['GET'])
--- a/ckan/controllers/api.py Wed Sep 28 11:45:57 2011 +0100
+++ b/ckan/controllers/api.py Wed Sep 28 19:06:06 2011 +0100
@@ -7,7 +7,7 @@
from ckan.lib.helpers import json, date_str_to_datetime
import ckan.model as model
import ckan.rating
-from ckan.lib.search import query_for, QueryOptions, SearchIndexError, SearchError, DEFAULT_OPTIONS
+from ckan.lib.search import query_for, QueryOptions, SearchIndexError, SearchError, DEFAULT_OPTIONS, convert_legacy_parameters_to_solr
from ckan.plugins import PluginImplementations, IGroupController
from ckan.lib.munge import munge_title_to_name
from ckan.lib.navl.dictization_functions import DataError
@@ -412,7 +412,7 @@
return self._finish_ok([rev.id for rev in revs])
elif register in ['dataset', 'package', 'resource']:
try:
- params = dict(self._get_search_params(request.params))
+ params = MultiDict(self._get_search_params(request.params))
except ValueError, e:
return self._finish_bad_request(
gettext('Could not read parameters: %r' % e))
@@ -451,7 +451,11 @@
query=params.get('q'), fields=query_fields, options=options
)
else:
- # for package searches we can pass parameters straight to Solr
+ # For package searches in API v3 and higher, we can pass
+ # parameters straight to Solr.
+ if ver in u'12':
+ # Otherwise, put all unrecognised ones into the q parameter
+ params = convert_legacy_parameters_to_solr(params)
query = query_for(model.Package)
results = query.run(params)
return self._finish_ok(results)
--- a/ckan/lib/search/__init__.py Wed Sep 28 11:45:57 2011 +0100
+++ b/ckan/lib/search/__init__.py Wed Sep 28 19:06:06 2011 +0100
@@ -7,7 +7,7 @@
from ckan.lib.dictization.model_dictize import package_to_api1
from common import SearchIndexError, SearchError, make_connection, is_available, DEFAULT_SOLR_URL
from index import PackageSearchIndex, NoopSearchIndex
-from query import TagSearchQuery, ResourceSearchQuery, PackageSearchQuery, QueryOptions
+from query import TagSearchQuery, ResourceSearchQuery, PackageSearchQuery, QueryOptions, convert_legacy_parameters_to_solr
log = logging.getLogger(__name__)
--- a/ckan/lib/search/query.py Wed Sep 28 11:45:57 2011 +0100
+++ b/ckan/lib/search/query.py Wed Sep 28 19:06:06 2011 +0100
@@ -1,6 +1,7 @@
import json
from pylons import config
from paste.deploy.converters import asbool
+from paste.util.multidict import MultiDict
from ckan import model
from ckan.logic import get_action
from common import make_connection, SearchError
@@ -19,14 +20,52 @@
# and their relative weighting
QUERY_FIELDS = "name^4 title^4 tags^2 groups^2 text"
+def convert_legacy_parameters_to_solr(legacy_params):
+ '''API v1 and v2 allowed search params that the SOLR syntax does not
+ support, so use this function to convert those to SOLR syntax.
+ See tests for examples.
+
+ raises ValueError on invalid params.
+ '''
+ options = QueryOptions(**legacy_params)
+ options.validate()
+ solr_params = legacy_params.copy()
+ solr_q_list = []
+ if solr_params.get('q'):
+ solr_q_list.append(solr_params['q'].replace('+', ' '))
+ non_solr_params = set(legacy_params.keys()) - VALID_SOLR_PARAMETERS
+ for search_key in non_solr_params:
+ value = str(legacy_params[search_key]).replace('+', ' ')
+ if search_key == 'all_fields':
+ if value:
+ solr_params['fl'] = '*'
+ elif search_key == 'offset':
+ solr_params['start'] = value
+ elif search_key == 'limit':
+ solr_params['rows'] = value
+ elif search_key == 'order_by':
+ solr_params['sort'] = '%s asc' % value
+ else:
+ if ' ' in value:
+ value = '"%s"' % value
+ solr_q_list.append('%s:%s' % (search_key, value))
+ del solr_params[search_key]
+ solr_params['q'] = ' '.join(solr_q_list)
+ if non_solr_params:
+ log.info('Converted legacy search params from %r to %r',
+ legacy_params, solr_params)
+ return solr_params
+
+
class QueryOptions(dict):
"""
Options specify aspects of the search query which are only tangentially related
to the query terms (such as limits, etc.).
"""
- BOOLEAN_OPTIONS = ['filter_by_downloadable', 'filter_by_openness', 'all_fields']
+ BOOLEAN_OPTIONS = ['all_fields']
INTEGER_OPTIONS = ['offset', 'limit']
+ UNSUPPORTED_OPTIONS = ['filter_by_downloadable', 'filter_by_openness']
def __init__(self, **kwargs):
from ckan.lib.search import DEFAULT_OPTIONS
@@ -50,6 +89,8 @@
value = int(value)
except ValueError:
raise SearchError('Value for search option %r must be an integer but received %r' % (key, value))
+ elif key in self.UNSUPPORTED_OPTIONS:
+ raise SearchError('Search option %r is not supported' % key)
self[key] = value
def __getattr__(self, name):
@@ -173,8 +214,7 @@
return [r.get('id') for r in data.results]
def run(self, query):
- assert isinstance(query, dict)
-
+ assert isinstance(query, (dict, MultiDict))
# check that query keys are valid
if not set(query.keys()) <= VALID_SOLR_PARAMETERS:
invalid_params = [s for s in set(query.keys()) - VALID_SOLR_PARAMETERS]
--- a/ckan/templates/layout_base.html Wed Sep 28 11:45:57 2011 +0100
+++ b/ckan/templates/layout_base.html Wed Sep 28 19:06:06 2011 +0100
@@ -101,7 +101,7 @@
<content><p>Master content template placeholder … please replace me.</p></content>
- </div>
+ </div><!-- /content --><div id="sidebar" class="span-7 last"><ul class="widget-list"><py:if test="defined('primary_sidebar_extras')">
--- a/ckan/tests/functional/api/base.py Wed Sep 28 11:45:57 2011 +0100
+++ b/ckan/tests/functional/api/base.py Wed Sep 28 19:06:06 2011 +0100
@@ -31,8 +31,6 @@
extra_environ = {}
api_version = None
- ref_package_by = ''
- ref_group_by = ''
def get(self, offset, status=[200]):
response = self.app.get(offset, status=status,
@@ -68,6 +66,87 @@
base += '/' + self.api_version
return '%s%s' % (base, path)
+ def assert_msg_represents_anna(self, msg):
+ assert 'annakarenina' in msg, msg
+ data = self.loads(msg)
+ self.assert_equal(data['name'], 'annakarenina')
+ self.assert_equal(data['license_id'], 'other-open')
+ assert '"license_id": "other-open"' in msg, str(msg)
+ assert 'russian' in msg, msg
+ assert 'tolstoy' in msg, msg
+ assert '"extras": {' in msg, msg
+ assert '"genre": "romantic novel"' in msg, msg
+ assert '"original media": "book"' in msg, msg
+ assert 'annakarenina.com/download' in msg, msg
+ assert '"plain text"' in msg, msg
+ assert '"Index of the novel"' in msg, msg
+ assert '"id": "%s"' % self.anna.id in msg, msg
+ expected = '"groups": ['
+ assert expected in msg, (expected, msg)
+ expected = self.group_ref_from_name('roger')
+ assert expected in msg, (expected, msg)
+ expected = self.group_ref_from_name('david')
+ assert expected in msg, (expected, msg)
+
+ # Todo: What is the deal with ckan_url? And should this use IDs rather than names?
+ assert 'ckan_url' in msg
+ assert '"ckan_url": "http://test.ckan.net/dataset/annakarenina"' in msg, msg
+
+ def assert_msg_represents_roger(self, msg):
+ assert 'roger' in msg, msg
+ data = self.loads(msg)
+ keys = set(data.keys())
+ expected_keys = set(['id', 'name', 'title', 'description', 'created',
+ 'state', 'revision_id', 'packages'])
+ missing_keys = expected_keys - keys
+ assert not missing_keys, missing_keys
+ assert_equal(data['name'], 'roger')
+ assert_equal(data['title'], 'Roger\'s books')
+ assert_equal(data['description'], 'Roger likes these books.')
+ assert_equal(data['state'], 'active')
+ assert_equal(data['packages'], [self._ref_package(self.anna)])
+
+ def assert_msg_represents_russian(self, msg):
+ data = self.loads(msg)
+ pkgs = set(data)
+ expected_pkgs = set([self.package_ref_from_name('annakarenina'),
+ self.package_ref_from_name('warandpeace')])
+ differences = expected_pkgs ^ pkgs
+ assert not differences, '%r != %r' % (pkgs, expected_pkgs)
+
+ def data_from_res(self, res):
+ return self.loads(res.body)
+
+ def get_expected_api_version(self):
+ return self.api_version
+
+ def dumps(self, data):
+ return json.dumps(data)
+
+ def loads(self, chars):
+ try:
+ return json.loads(chars)
+ except ValueError, inst:
+ raise Exception, "Couldn't loads string '%s': %s" % (chars, inst)
+
+ def assert_json_response(self, res, expected_in_body=None):
+ content_type = res.header_dict['Content-Type']
+ assert 'application/json' in content_type, content_type
+ res_json = self.loads(res.body)
+ if expected_in_body:
+ assert expected_in_body in res_json or \
+ expected_in_body in str(res_json), \
+ 'Expected to find %r in JSON response %r' % \
+ (expected_in_body, res_json)
+
+class Api1and2TestCase(object):
+ ''' Utils for v1 and v2 API.
+ * RESTful URL utils
+ * handling of different ways of referencing packages (by name or id)
+ '''
+ ref_package_by = ''
+ ref_group_by = ''
+
def package_offset(self, package_name=None):
if package_name is None:
# Package Register
@@ -197,81 +276,7 @@
def _list_group_refs(cls, groups):
return [getattr(p, cls.ref_group_by) for p in groups]
- def assert_msg_represents_anna(self, msg):
- assert 'annakarenina' in msg, msg
- data = self.loads(msg)
- self.assert_equal(data['name'], 'annakarenina')
- self.assert_equal(data['license_id'], 'other-open')
- assert '"license_id": "other-open"' in msg, str(msg)
- assert 'russian' in msg, msg
- assert 'tolstoy' in msg, msg
- assert '"extras": {' in msg, msg
- assert '"genre": "romantic novel"' in msg, msg
- assert '"original media": "book"' in msg, msg
- assert 'annakarenina.com/download' in msg, msg
- assert '"plain text"' in msg, msg
- assert '"Index of the novel"' in msg, msg
- assert '"id": "%s"' % self.anna.id in msg, msg
- expected = '"groups": ['
- assert expected in msg, (expected, msg)
- expected = self.group_ref_from_name('roger')
- assert expected in msg, (expected, msg)
- expected = self.group_ref_from_name('david')
- assert expected in msg, (expected, msg)
-
- # Todo: What is the deal with ckan_url? And should this use IDs rather than names?
- assert 'ckan_url' in msg
- assert '"ckan_url": "http://test.ckan.net/dataset/annakarenina"' in msg, msg
-
- def assert_msg_represents_roger(self, msg):
- assert 'roger' in msg, msg
- data = self.loads(msg)
- keys = set(data.keys())
- expected_keys = set(['id', 'name', 'title', 'description', 'created',
- 'state', 'revision_id', 'packages'])
- missing_keys = expected_keys - keys
- assert not missing_keys, missing_keys
- assert_equal(data['name'], 'roger')
- assert_equal(data['title'], 'Roger\'s books')
- assert_equal(data['description'], 'Roger likes these books.')
- assert_equal(data['state'], 'active')
- assert_equal(data['packages'], [self._ref_package(self.anna)])
-
- def assert_msg_represents_russian(self, msg):
- data = self.loads(msg)
- pkgs = set(data)
- expected_pkgs = set([self.package_ref_from_name('annakarenina'),
- self.package_ref_from_name('warandpeace')])
- differences = expected_pkgs ^ pkgs
- assert not differences, '%r != %r' % (pkgs, expected_pkgs)
-
- def data_from_res(self, res):
- return self.loads(res.body)
-
- def get_expected_api_version(self):
- return self.api_version
-
- def dumps(self, data):
- return json.dumps(data)
-
- def loads(self, chars):
- try:
- return json.loads(chars)
- except ValueError, inst:
- raise Exception, "Couldn't loads string '%s': %s" % (chars, inst)
-
- def assert_json_response(self, res, expected_in_body=None):
- content_type = res.header_dict['Content-Type']
- assert 'application/json' in content_type, content_type
- res_json = self.loads(res.body)
- if expected_in_body:
- assert expected_in_body in res_json or \
- expected_in_body in str(res_json), \
- 'Expected to find %r in JSON response %r' % \
- (expected_in_body, res_json)
-
-# Todo: Rename to Version1TestCase.
-class Api1TestCase(ApiTestCase):
+class Api1TestCase(Api1and2TestCase):
api_version = '1'
ref_package_by = 'name'
@@ -283,7 +288,7 @@
assert '"download_url": "http://www.annakarenina.com/download/x=1&y=2"' in msg, msg
-class Api2TestCase(ApiTestCase):
+class Api2TestCase(Api1and2TestCase):
api_version = '2'
ref_package_by = 'id'
@@ -295,6 +300,17 @@
assert 'download_url' not in msg, msg
+class Api3TestCase(ApiTestCase):
+
+ api_version = '3'
+ ref_package_by = 'id'
+ ref_group_by = 'id'
+ ref_tag_by = 'id'
+
+ def assert_msg_represents_anna(self, msg):
+ super(Api2TestCase, self).assert_msg_represents_anna(msg)
+ assert 'download_url' not in msg, msg
+
class ApiUnversionedTestCase(Api1TestCase):
api_version = ''
--- a/ckan/tests/functional/api/test_package_search.py Wed Sep 28 11:45:57 2011 +0100
+++ b/ckan/tests/functional/api/test_package_search.py Wed Sep 28 19:06:06 2011 +0100
@@ -1,4 +1,5 @@
from nose.tools import assert_raises
+from nose.plugins.skip import SkipTest
from ckan import plugins
import ckan.lib.search as search
@@ -159,21 +160,6 @@
self.assert_results(res_dict, [u'annakarenina', u'warandpeace'])
assert res_dict['count'] == 2, res_dict
- def test_07_uri_qjson_extras(self):
- # TODO: solr is not currently set up to allow partial matches
- # and extras are not saved as multivalued so this
- # test will fail. Make extras multivalued or remove?
- from ckan.tests import SkipTest
- raise SkipTest
-
- query = {"geographic_coverage":"England"}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_results(res_dict, ['testpkg'])
- assert res_dict['count'] == 1, res_dict
-
def test_07_uri_qjson_extras_2(self):
query = {'q': "national_statistic:yes"}
json_query = self.dumps(query)
@@ -188,108 +174,23 @@
res = self.app.get(offset, status=400)
self.assert_json_response(res, 'Bad request - Could not read parameters')
- def test_08_all_fields(self):
- rating = model.Rating(user_ip_address=u'123.1.2.3',
- package=self.anna,
- rating=3.0)
- model.Session.add(rating)
- model.repo.commit_and_remove()
-
- query = {'q': 'russian', 'fl': '*'}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 2, res_dict
- for rec in res_dict['results']:
- if rec['name'] == 'annakarenina':
- anna_rec = rec
- break
- assert anna_rec['name'] == 'annakarenina', res_dict['results']
- assert anna_rec['title'] == 'A Novel By Tolstoy', anna_rec['title']
- assert anna_rec['license_id'] == u'other-open', anna_rec['license_id']
- assert len(anna_rec['tags']) == 2, anna_rec['tags']
- for expected_tag in ['russian', 'tolstoy']:
- assert expected_tag in anna_rec['tags']
-
- # TODO: these values are not being passed to Solr
- # assert anna_rec['ratings_average'] == 3.0, anna_rec['ratings_average']
- # assert anna_rec['ratings_count'] == 1, anna_rec['ratings_count']
-
- # try alternative syntax
- offset = self.base_url + '?q=russian&fl=*'
- res2 = self.app.get(offset, status=200)
- assert_equal(res2.body, res.body)
-
def test_08_all_fields_syntax_error(self):
offset = self.base_url + '?all_fields=should_be_boolean' # invalid all_fields value
res = self.app.get(offset, status=400)
assert('all_fields' in res.body)
def test_09_just_tags(self):
- offset = self.base_url + '?q=tags:russian&fl=*'
+ offset = self.base_url + '?q=tags:russian'
res = self.app.get(offset, status=200)
res_dict = self.data_from_res(res)
assert res_dict['count'] == 2, res_dict
def test_10_multiple_tags(self):
- offset = self.base_url + '?q=tags:tolstoy tags:russian&fl=*'
+ offset = self.base_url + '?q=tags:tolstoy tags:russian'
res = self.app.get(offset, status=200)
res_dict = self.data_from_res(res)
assert res_dict['count'] == 1, res_dict
- def test_10_multiple_tags_with_plus(self):
- # TODO: this syntax doesn't work with Solr search, update documentation
- from nose import SkipTest
- raise SkipTest
-
- offset = self.base_url + '?tags=tolstoy+russian&all_fields=1'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
- def test_10_multiple_tags_with_ampersand(self):
- # TODO: this syntax doesn't work with Solr search, update documentation
- from nose import SkipTest
- raise SkipTest
-
- offset = self.base_url + '?tags=tolstoy&tags=russian&all_fields=1'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
- def test_10_many_tags_with_ampersand(self):
- # TODO: this syntax doesn't work with Solr search, update documentation
- from nose import SkipTest
- raise SkipTest
-
- offset = self.base_url + '?tags=tolstoy&tags=russian&tags=tolstoy'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
- def test_11_pagination_limit(self):
- offset = self.base_url + '?fl=*&q=tags:russian&rows=1&sort=name asc'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 2, res_dict
- assert len(res_dict['results']) == 1, res_dict
- assert res_dict['results'][0]['name'] == 'annakarenina', res_dict['results'][0]['name']
-
- def test_11_pagination_offset_limit(self):
- offset = self.base_url + '?fl=*&q=tags:russian&start=1&rows=1&sort=name asc'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 2, res_dict
- assert len(res_dict['results']) == 1, res_dict
- assert res_dict['results'][0]['name'] == 'warandpeace', res_dict['results'][0]['name']
-
- def test_11_pagination_syntax_error(self):
- offset = self.base_url + '?fl=*&q="tags:russian"&start=should_be_integer&rows=1&sort=name' # invalid offset value
- res = self.app.get(offset, status=400)
- print res.body
- assert('should_be_integer' in res.body)
-
def test_12_all_packages_qjson(self):
query = {'q': ''}
json_query = self.dumps(query)
@@ -310,15 +211,174 @@
res_dict = self.data_from_res(res)
assert_equal(res_dict['count'], 3)
+ def test_12_filter_by_openness_qjson(self):
+ query = {'q': '', 'filter_by_openness': '1'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=400)
+
+ def test_12_filter_by_openness_q(self):
+ offset = self.base_url + '?filter_by_openness=1'
+ res = self.app.get(offset, status=400)
+
def test_13_just_groups(self):
offset = self.base_url + '?q=groups:roger'
res = self.app.get(offset, status=200)
res_dict = self.data_from_res(res)
assert res_dict['count'] == 1, res_dict
-class TestPackageSearchApi1(Api1TestCase, PackageSearchApiTestCase): pass
-class TestPackageSearchApi2(Api2TestCase, PackageSearchApiTestCase): pass
-class TestPackageSearchApiUnversioned(PackageSearchApiTestCase, ApiUnversionedTestCase): pass
+class LegacyOptionsTestCase(ApiTestCase, ControllerTestCase):
+ @classmethod
+ def setup_class(self):
+ setup_test_search_index()
+ CreateTestData.create()
+ self.package_fixture_data = {
+ 'name' : u'testpkg',
+ 'title': 'Some Title',
+ 'url': u'http://blahblahblah.mydomain',
+ 'resources': [{u'url':u'http://blahblahblah.mydomain',
+ u'format':u'', u'description':''}],
+ 'tags': ['russion', 'novel'],
+ 'license_id': u'gpl-3.0',
+ 'extras': {'national_statistic':'yes',
+ 'geographic_coverage':'England, Wales'},
+ }
+ CreateTestData.create_arbitrary(self.package_fixture_data)
+ self.base_url = self.offset('/search/dataset')
+ @classmethod
+ def teardown_class(cls):
+ model.repo.rebuild_db()
+ search.clear()
+ def test_07_uri_qjson_extras(self):
+ # TODO: solr is not currently set up to allow partial matches
+ # and extras are not saved as multivalued so this
+ # test will fail. Make extras multivalued or remove?
+ raise SkipTest()
+
+ query = {"geographic_coverage":"England"}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict
+ def test_08_all_fields(self):
+ rating = model.Rating(user_ip_address=u'123.1.2.3',
+ package=self.anna,
+ rating=3.0)
+ model.Session.add(rating)
+ model.repo.commit_and_remove()
+
+ query = {'q': 'russian', 'all_fields': '1'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ for rec in res_dict['results']:
+ if rec['name'] == 'annakarenina':
+ anna_rec = rec
+ break
+ assert anna_rec['name'] == 'annakarenina', res_dict['results']
+ assert anna_rec['title'] == 'A Novel By Tolstoy', anna_rec['title']
+ assert anna_rec['license_id'] == u'other-open', anna_rec['license_id']
+ assert len(anna_rec['tags']) == 2, anna_rec['tags']
+ for expected_tag in ['russian', 'tolstoy']:
+ assert expected_tag in anna_rec['tags']
+
+ # try alternative syntax
+ offset = self.base_url + '?q=russian&all_fields=1'
+ res2 = self.app.get(offset, status=200)
+ assert_equal(res2.body, res.body)
+
+ def test_09_tags(self):
+ offset = self.base_url + '?tags=tolstoy'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_10_multiple_tags_with_ampersand(self):
+ offset = self.base_url + '?tags=tolstoy&tags=russian&all_fields=1'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_10_many_tags_with_ampersand(self):
+ offset = self.base_url + '?tags=tolstoy&tags=russian&tags=tolstoy'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_11_pagination_limit(self):
+ offset = self.base_url + '?all_fields=1&q=tags:russian&limit=1&order_by=name'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ assert len(res_dict['results']) == 1, res_dict
+ assert res_dict['results'][0]['name'] == 'annakarenina', res_dict['results'][0]['name']
+
+ def test_11_pagination_offset_limit(self):
+ offset = self.base_url + '?all_fields=1&q=tags:russian&offset=1&limit=1&order_by=name'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ assert len(res_dict['results']) == 1, res_dict
+ assert res_dict['results'][0]['name'] == 'warandpeace', res_dict['results'][0]['name']
+
+ def test_11_pagination_syntax_error(self):
+ offset = self.base_url + '?all_fields=1&q="tags:russian"&start=should_be_integer&rows=1&order_by=name' # invalid offset value
+ res = self.app.get(offset, status=400)
+ print res.body
+ assert('should_be_integer' in res.body)
+
+class TestPackageSearchApi1(Api1TestCase, PackageSearchApiTestCase,
+ LegacyOptionsTestCase): pass
+class TestPackageSearchApi2(Api2TestCase, PackageSearchApiTestCase,
+ LegacyOptionsTestCase): pass
+class TestPackageSearchApi3(Api3TestCase, PackageSearchApiTestCase):
+ def test_08_all_fields(self):
+ query = {'q': 'russian', 'fl': '*'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ for rec in res_dict['results']:
+ if rec['name'] == 'annakarenina':
+ anna_rec = rec
+ break
+ assert anna_rec['name'] == 'annakarenina', res_dict['results']
+ assert anna_rec['title'] == 'A Novel By Tolstoy', anna_rec['title']
+ assert anna_rec['license_id'] == u'other-open', anna_rec['license_id']
+ assert len(anna_rec['tags']) == 2, anna_rec['tags']
+ for expected_tag in ['russian', 'tolstoy']:
+ assert expected_tag in anna_rec['tags']
+
+ # try alternative syntax
+ offset = self.base_url + '?q=russian&fl=*'
+ res2 = self.app.get(offset, status=200)
+ assert_equal(res2.body, res.body)
+
+ def test_09_just_tags(self):
+ offset = self.base_url + '?q=tags:russian&fl=*'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+
+ def test_11_pagination_limit(self):
+ offset = self.base_url + '?fl=*&q=tags:russian&rows=1&sort=name asc'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ assert len(res_dict['results']) == 1, res_dict
+ assert res_dict['results'][0]['name'] == 'annakarenina', res_dict['results'][0]['name']
+
+class TestPackageSearchApiUnversioned(PackageSearchApiTestCase,
+ ApiUnversionedTestCase,
+ LegacyOptionsTestCase): pass
+
+
+
--- a/ckan/tests/lib/test_solr_package_search.py Wed Sep 28 11:45:57 2011 +0100
+++ b/ckan/tests/lib/test_solr_package_search.py Wed Sep 28 19:06:06 2011 +0100
@@ -1,8 +1,25 @@
+from nose.tools import assert_equal, assert_raises
+
from ckan.tests import TestController, CreateTestData, setup_test_search_index
from ckan import model
import ckan.lib.search as search
+class TestQuery:
+ def test_1_convert_legacy_params_to_solr(self):
+ convert = search.convert_legacy_parameters_to_solr
+ assert_equal(convert({'title': 'bob'}), {'q': 'title:bob'})
+ assert_equal(convert({'title': 'bob', 'fl': 'name'}),
+ {'q': 'title:bob', 'fl': 'name'})
+ assert_equal(convert({'title': 'bob perkins'}), {'q': 'title:"bob perkins"'})
+ assert_equal(convert({'q': 'high+wages'}), {'q': 'high wages'})
+ assert_equal(convert({'q': 'high+wages summary'}), {'q': 'high wages summary'})
+ assert_equal(convert({'title': 'high+wages'}), {'q': 'title:"high wages"'})
+ assert_equal(convert({'title': 'bob', 'all_fields': 1}), {'q': 'title:bob', 'fl': '*'})
+ assert_raises(search.SearchError, convert, {'title': 'bob', 'all_fields': 'non-boolean'})
+ assert_equal(convert({'q': 'bob', 'order_by': 'name'}), {'q': 'bob', 'sort':'name asc'})
+ assert_equal(convert({'q': 'bob', 'offset': '0', 'limit': '10'}), {'q': 'bob', 'start':'0', 'rows':'10'})
+
class TestSearch(TestController):
# 'penguin' is in all test search packages
q_all = u'penguin'
--- a/doc/action_api.rst Wed Sep 28 11:45:57 2011 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,241 +0,0 @@
-.. index:: API
-
-===========================
-Reference: CKAN Action API
-===========================
-
-.. toctree::
- :hidden:
- :maxdepth: 1
-
-.. warning:: The Action API is still experimental and subject to change of URI locations, formats, parameters and results.
-
-Overview
---------
-
-The Action API is a powerful RPC-style way of accessing CKAN data. Its intention is to have access to all the core logic in ckan. It calls exactly the same functions that are used internally which all the other CKAN interfaces (Web interface / Model API) go through. Therefore it provides the full gamut of read and write operations, with all possible parameters.
-
-A client supplies parameters to the Action API via a JSON dictionary of a POST request, and returns results, help information and any error diagnostics in a JSON dictionary too. This is a departure from the CKAN API versions 1 and 2, which being RESTful required all the request parameters to be part of the URL.
-
-Requests
---------
-
-URL
-===
-
-The basic URL for the Action API is::
-
- /api/action/{logic_action}
-
-Examples::
-
- /api/action/package_list
- /api/action/package_show
- /api/action/user_create
-
-Actions
-=======
-
-get.py:
-
-====================================== ===========================
-Logic Action Parameter keys
-====================================== ===========================
-site_read (none)
-package_list (none)
-current_package_list_with_resources limit
-revision_list (none)
-package_revision_list id
-group_list all_fields
-group_list_authz (none)
-group_list_available (none)
-group_revision_list id
-licence_list (none)
-tag_list q, all_fields, limit, offset, return_objects
-user_list q, order_by
-package_relationships_list id, id2, rel
-package_show id
-revision_show id
-group_show id
-tag_show id
-user_show id
-package_show_rest id
-group_show_rest id
-tag_show_rest id
-package_autocomplete q
-tag_autocomplete q, limit
-format_autocomplete q, limit
-user_autocomplete q, limit
-package_search q, fields, facet_by, limit, offset, filter_by_openness, filter_by_downloadable
-====================================== ===========================
-
-new.py:
-
-====================================== ===========================
-Logic Action Parameter keys
-====================================== ===========================
-package_create (package keys)
-package_create_validate (package keys)
-resource_create (resource keys)
-package_relationship_create id, id2, rel, comment
-group_create (group keys)
-rating_create package, rating
-user_create (user keys)
-package_create_rest (package keys)
-group_create_rest (group keys)
-====================================== ===========================
-
-update.py:
-
-====================================== ===========================
-Logic Action Parameter keys
-====================================== ===========================
-make_latest_pending_package_active id
-resource_update (resource keys)
-package_update (package keys)
-package_update_validate (package keys)
-package_relationship_update id, id2, rel, comment
-group_update (group keys)
-user_update (user keys), reset_key
-package_update_rest (package keys)
-group_update_rest (group keys)
-====================================== ===========================
-
-delete.py:
-
-====================================== ===========================
-Logic Action Parameter keys
-====================================== ===========================
-package_delete id
-package_relationship_delete id, id2, rel
-group_delete id
-====================================== ===========================
-
-In case of doubt, refer to the code of the logic actions, which is found in the CKAN source in the ckan/logic/action directory.
-
-Object dictionaries
-===================
-
-Package:
-
-======================== ====================================== =============
-key example value notes
-======================== ====================================== =============
-id "fd788e57-dce4-481c-832d-497235bf9f78" (Read-only) unique identifier
-name "uk-spending" Unique identifier. Should be human readable
-title "UK Spending" Human readable title of the dataset
-url "http://gov.uk/spend-downloads.html" Home page for the data
-version "1.0" Version associated with the data. String format.
-author "UK Treasury" Name of person responsible for the data
-author_email "contact at treasury.gov.uk" Email address for the person in the 'author' field
-maintainer null Name of another person responsible for the data
-maintainer_email null Email address for the person in the 'maintainer' field
-notes "### About\\r\\n\\r\\nUpdated 1997." Other human readable info about the dataset. Markdown format.
-license_id "cc-by" ID of the license this dataset is released under. You can then look up the license ID to get the title.
-extras []
-tags ["government-spending"] List of tags associated with this dataset.
-groups ["spending", "country-uk"] List of groups this dataset is a member of.
-relationships_as_subject [] List of relationships (edit this only using relationship specific command). The 'type' of the relationship is described in terms of this package being the subject and the related package being the object.
-state active May be ``deleted`` or other custom states like ``pending``.
-revision_id "f645243a-7334-44e2-b87c-64231700a9a6" (Read-only) ID of the last revision for the core package object was (doesn't include tags, groups, extra fields, relationships).
-revision_timestamp "2010-12-21T15:26:17.345502" (Read-only) Time and date when the last revision for the core package object was (doesn't include tags, groups, extra fields, relationships). ISO format. UTC timezone assumed.
-======================== ====================================== =============
-
-Package Extra:
-
-======================== ====================================== =============
-key example value notes
-======================== ====================================== =============
-id "c10fb749-ad46-4ba2-839a-41e8e2560687" (Read-only)
-key "number_of_links"
-value "10000"
-package_id "349259a8-cbff-4610-8089-2c80b34e27c5" (Read-only) Edit package extras with package_update
-state "active" (Read-only) Edit package extras with package_update
-revision_timestamp "2010-09-01T08:56:53.696551" (Read-only)
-revision_id "233d0c19-fcdc-44b9-9afe-25e2aa9d0a5f" (Read-only)
-======================== ====================================== =============
-
-
-Resource:
-
-======================== ====================================== =============
-key example value notes
-======================== ====================================== =============
-id "888d00e9-6ee5-49ca-9abb-6f216e646345" (Read-only)
-url "http://gov.uk/spend-july-2009.csv" Download URL of the data
-description ""
-format "XLS" Format of the data
-hash null Hash of the data e.g. SHA1
-state "active"
-position 0 (Read-only) This is set by the order of resources are given in the list when creating/updating the package.
-resource_group_id "49ddadb0-dd80-9eff-26e9-81c5a466cf6e" (Read-only)
-revision_id "188ac88b-1573-48bf-9ea6-d3c503db5816" (Read-only)
-revision_timestamp "2011-07-08T14:48:38.967741" (Read-only)
-======================== ====================================== =============
-
-Tag:
-
-======================== ====================================== =============
-key example value notes
-======================== ====================================== =============
-id "b10871ea-b4ae-4e2e-bec9-a8d8ff357754" (Read-only)
-name "country-uk" (Read-only) Add/remove tags from a package or group using update_package or update_group
-state "active" (Read-only) Add/remove tags from a package or group using update_package or update_group
-revision_timestamp "2009-08-08T12:46:40.920443" (Read-only)
-======================== ====================================== =============
-
-Parameters
-==========
-
-Requests must be a POST, including parameters in a JSON dictionary. If there are no parameters required, then an empty dictionary is still required (or you get a 400 error).
-
-Examples::
-
- curl http://test.ckan.net/api/action/package_list -d '{}'
- curl http://test.ckan.net/api/action/package_show -d '{"id": "fd788e57-dce4-481c-832d-497235bf9f78"}'
-
-Authorization Header
-====================
-
-Authorization is carried out the same way as the existing API, supplying the user's API key in the "Authorization" header.
-
-Depending on the settings of the instance, you may not need to identify yourself for simple read operations. (This is the case for thedatahub.org and is assumed for the examples below.)
-
-JSONP
-=====
-
-TBC
-
-Responses
-=========
-
-The response is wholly contained in the form of a JSON dictionary. Here is the basic format of a successful request::
-
- {"help": "Creates a package", "success": true, "result": ...}
-
-And here is one that incurred an error::
-
- {"help": "Creates a package", "success": false, "error": {"message": "Access denied", "__type": "Authorization Error"}}
-
-Where:
-
-* ``help`` is the 'doc string' (or ``null``)
-* ``success`` is ``true`` or ``false`` depending on whether the request was successful. The response is always status 200, so it is important to check this value.
-* ``result`` is the main payload that results from a successful request. This might be a list of the domain object names or a dictionary with the particular domain object.
-* ``error`` is supplied if the request was not successful and provides a message and __type. See the section on errors.
-
-Errors
-======
-
-The message types include:
- * Authorization Error - an API key is required for this operation, and the corresponding user needs the correct credentials
- * Validation Error - the object supplied does not meet with the standards described in the schema.
- * (TBC) JSON Error - the request could not be parsed / decoded as JSON format, according to the Content-Type (default is ``application/x-www-form-urlencoded;utf-8``).
-
-Examples
-========
-
-::
-
- $ curl http://ckan.net/api/action/package_show -d '{"id": "fd788e57-dce4-481c-832d-497235bf9f78"}'
- {"help": null, "success": true, "result": {"maintainer": null, "name": "uk-quango-data", "relationships_as_subject": [], "author": null, "url": "http://www.guardian.co.uk/news/datablog/2009/jul/07/public-finance-regulators", "relationships_as_object": [], "notes": "### About\r\n\r\nDid you know there are nearly 1,200 unelected bodies with power over our lives? This is the full list, complete with number of staff and how much they cost. As a spreadsheet\r\n\r\n### Openness\r\n\r\nNo licensing information found.", "title": "Every Quango in Britain", "maintainer_email": null, "revision_timestamp": "2010-12-21T15:26:17.345502", "author_email": null, "state": "active", "version": null, "groups": [], "license_id": "notspecified", "revision_id": "f645243a-7334-44e2-b87c-64231700a9a6", "tags": [{"revision_timestamp": "2009-08-08T12:46:40.920443", "state": "active", "id": "b10871ea-b4ae-4e2e-bec9-a8d8ff357754", "name": "country-uk"}, {"revision_timestamp": "2009-08-08T12:46:40.920443", "state": "active", "id": "ed783bc3-c0a1-49f6-b861-fd9adbc1006b", "name": "quango"}], "id": "fd788e57-dce4-481c-832d-497235bf9f78", "resources": [{"resource_group_id": "49ddadb0-dd80-9eff-26e9-81c5a466cf6e", "hash": null, "description": "", "format": "", "url": "http://spreadsheets.google.com/ccc?key=tm4Dxoo0QtDrEOEC1FAJuUg", "revision_timestamp": "2011-07-08T14:48:38.967741", "state": "active", "position": 0, "revision_id": "188ac88b-1573-48bf-9ea6-d3c503db5816", "id": "888d00e9-6ee5-49ca-9abb-6f216e646345"}], "extras": []}}
\ No newline at end of file
--- a/doc/api.rst Wed Sep 28 11:45:57 2011 +0100
+++ b/doc/api.rst Wed Sep 28 19:06:06 2011 +0100
@@ -59,6 +59,12 @@
The only exception for this is for Tag objects. Since Tag names are immutable, they are always referred to with their Name.
+Version 3
+~~~~~~~~~
+
+This version is in beta. To details of trying it out, see :doc:`apiv3`.
+
+
API Details - Versions 1 & 2
----------------------------
@@ -85,7 +91,6 @@
* `Model API`_
* `Search API`_
* `Util API`_
-* `Action API`_
The resources, methods, and data formats of each are described below.
@@ -329,7 +334,7 @@
| | | | parameter as a more flexible |
| | | | alternative in GET requests. |
+-----------------------+---------------+----------------------------------+----------------------------------+
-|title, | Search-String || title=uk&tags=health+census | Search in a particular a field. |
+|title, | Search-String || title=uk&tags=health | Search in a particular a field. |
|tags, notes, groups, | || department=environment | |
|author, maintainer, | | | |
|update_frequency, or | | | |
@@ -349,12 +354,9 @@
| | | | (0) or the full dataset record |
| | | | (1). |
+-----------------------+---------------+----------------------------------+----------------------------------+
-| filter_by_openness | 0 (default) | filter_by_openness=1 | Filters results by ones which are|
-| | or 1 | | open. |
-+-----------------------+---------------+----------------------------------+----------------------------------+
-|filter_by_downloadable | 0 (default) | filter_by_downloadable=1 | Filters results by ones which |
-| | or 1 | | have at least one resource URL. |
-+-----------------------+---------------+----------------------------------+----------------------------------+
+
+.. Note: filter_by_openness and filter_by_downloadable were dropped from CKAN version 1.5 onwards.
+
**Resource Parameters**
@@ -554,4 +556,4 @@
Action API
~~~~~~~~~~
-See: :doc:`action_api`
\ No newline at end of file
+See:
\ No newline at end of file
Repository URL: https://bitbucket.org/okfn/ckan/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list