[ckan-changes] commit/ckan: John Glover: [solr] [1154] Check for search indexing errors.
Bitbucket
commits-noreply at bitbucket.org
Thu Aug 25 14:26:51 UTC 2011
1 new changeset in ckan:
http://bitbucket.org/okfn/ckan/changeset/37a158fdfdf3/
changeset: 37a158fdfdf3
branch: feature-1275-solr-search
user: John Glover
date: 2011-08-25 16:26:35
summary: [solr] [1154] Check for search indexing errors.
Throw a SearchIndexError if indexing fails, catch in controllers and display a suitable error message. Test that adding/updating packages fails if Solr is down.
affected #: 7 files (3.0 KB)
--- a/ckan/controllers/api.py Wed Aug 24 15:56:31 2011 +0100
+++ b/ckan/controllers/api.py Thu Aug 25 15:26:35 2011 +0100
@@ -7,7 +7,7 @@
from ckan.lib.helpers import json
import ckan.model as model
import ckan.rating
-from ckan.lib.search import query_for, QueryOptions, SearchError, DEFAULT_OPTIONS
+from ckan.lib.search import query_for, QueryOptions, SearchIndexError, SearchError, DEFAULT_OPTIONS
from ckan.plugins import PluginImplementations, IGroupController
from ckan.lib.munge import munge_title_to_name
from ckan.lib.navl.dictization_functions import DataError
@@ -282,6 +282,9 @@
log.error('Format incorrect: %s' % request_data)
#TODO make better error message
return self._finish(400, _(u'Integrity Error') % request_data)
+ except SearchIndexError:
+ log.error('Unable to add package to search index: %s' % request_data)
+ return self._finish(500, _(u'Unable to add package to search index') % request_data)
except:
model.Session.rollback()
raise
@@ -328,6 +331,9 @@
log.error('Format incorrect: %s' % request_data)
#TODO make better error message
return self._finish(400, _(u'Integrity Error') % request_data)
+ except SearchIndexError:
+ log.error('Unable to update search index: %s' % request_data)
+ return self._finish(500, _(u'Unable to update search index') % request_data)
def delete(self, ver=None, register=None, subregister=None, id=None, id2=None):
action_map = {
--- a/ckan/controllers/package.py Wed Aug 24 15:56:31 2011 +0100
+++ b/ckan/controllers/package.py Thu Aug 25 15:26:35 2011 +0100
@@ -17,7 +17,7 @@
from ckan.lib.base import request, c, BaseController, model, abort, h, g, render
from ckan.lib.base import etag_cache, response, redirect, gettext
from ckan.authz import Authorizer
-from ckan.lib.search import SearchError
+from ckan.lib.search import SearchIndexError, SearchError
from ckan.lib.cache import proxy_cache
from ckan.lib.package_saver import PackageSaver, ValidationException
from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
@@ -473,6 +473,8 @@
abort(404, _('Package not found'))
except DataError:
abort(400, _(u'Integrity Error'))
+ except SearchIndexError:
+ abort(500, _(u'Unable to add package to search index.'))
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
@@ -506,6 +508,8 @@
abort(404, _('Package not found'))
except DataError:
abort(400, _(u'Integrity Error'))
+ except SearchIndexError:
+ abort(500, _(u'Unable to update search index.'))
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
--- a/ckan/lib/search/__init__.py Wed Aug 24 15:56:31 2011 +0100
+++ b/ckan/lib/search/__init__.py Thu Aug 25 15:26:35 2011 +0100
@@ -3,7 +3,7 @@
from ckan.model import DomainObjectOperation
from ckan.plugins import SingletonPlugin, implements, IDomainObjectModification
from ckan.lib.dictization.model_dictize import package_to_api1
-from common import SearchError, make_connection, is_available
+from common import SearchIndexError, SearchError, make_connection, is_available
from index import PackageSearchIndex, NoopSearchIndex
from query import TagSearchQuery, ResourceSearchQuery, PackageSearchQuery, QueryOptions
--- a/ckan/lib/search/common.py Wed Aug 24 15:56:31 2011 +0100
+++ b/ckan/lib/search/common.py Thu Aug 25 15:26:35 2011 +0100
@@ -3,8 +3,13 @@
import logging
log = logging.getLogger(__name__)
+class SearchIndexError(Exception): pass
class SearchError(Exception): pass
+solr_url = config.get('solr_url', 'http://127.0.0.1:8983/solr')
+solr_user = config.get('solr_user')
+solr_password = config.get('solr_password')
+
def is_available():
"""
Return true if we can successfully connect to Solr.
@@ -26,11 +31,7 @@
return config.get('search_enabled', True)
def make_connection(config):
- url = config.get('solr_url', 'http://localhost:8983/solr')
- user = config.get('solr_user')
- password = config.get('solr_password')
-
- if user is not None and password is not None:
- return SolrConnection(url, http_user=user, http_pass=password)
+ if solr_user is not None and solr_password is not None:
+ return SolrConnection(solr_url, http_user=solr_user, http_pass=solr_password)
else:
- return SolrConnection(url)
+ return SolrConnection(solr_url)
--- a/ckan/lib/search/index.py Wed Aug 24 15:56:31 2011 +0100
+++ b/ckan/lib/search/index.py Thu Aug 25 15:26:35 2011 +0100
@@ -1,7 +1,7 @@
from pylons import config
import itertools
import string
-from common import is_enabled, make_connection
+from common import SearchIndexError, is_enabled, make_connection
import logging
log = logging.getLogger(__name__)
@@ -84,8 +84,10 @@
def index_package(self, pkg_dict, config):
if (not is_enabled()) or (pkg_dict is None):
return
+
if (not pkg_dict.get('state')) or ('active' not in pkg_dict.get('state')):
return self.delete_package(pkg_dict, config)
+
conn = make_connection(config)
index_fields = RESERVED_FIELDS + pkg_dict.keys()
@@ -134,6 +136,9 @@
try:
conn.add_many([pkg_dict])
conn.commit(wait_flush=False, wait_searcher=False)
+ except Exception, e:
+ log.exception(e)
+ raise SearchIndexError(e)
finally:
conn.close()
@@ -150,5 +155,8 @@
try:
conn.delete_query(query)
conn.commit()
+ except Exception, e:
+ log.exception(e)
+ raise SearchIndexError(e)
finally:
conn.close()
--- a/ckan/tests/functional/api/model/test_package.py Wed Aug 24 15:56:31 2011 +0100
+++ b/ckan/tests/functional/api/model/test_package.py Thu Aug 25 15:26:35 2011 +0100
@@ -1,11 +1,13 @@
import copy
-from nose.tools import assert_equal
+from nose.tools import assert_equal, assert_raises
from ckan.tests.functional.api.base import BaseModelApiTestCase
from ckan.tests.functional.api.base import Api1TestCase as Version1TestCase
from ckan.tests.functional.api.base import Api2TestCase as Version2TestCase
from ckan.tests.functional.api.base import ApiUnversionedTestCase as UnversionedTestCase
+from ckan import plugins
+import ckan.lib.search as search
# Todo: Remove this ckan.model stuff.
import ckan.model as model
@@ -134,18 +136,6 @@
else:
assert package
- def test_register_post_json(self):
- assert not self.get_package_by_name(self.package_fixture_data['name'])
- offset = self.package_offset()
- data = self.dumps(self.package_fixture_data)
- res = self.post_json(offset, data, status=self.STATUS_201_CREATED,
- extra_environ=self.extra_environ)
- # Check the database record.
- self.remove()
- package = self.get_package_by_name(self.package_fixture_data['name'])
- assert package
- self.assert_equal(package.title, self.package_fixture_data['title'])
-
def test_register_post_bad_request(self):
test_params = {
'name':u'testpackage06_400',
@@ -173,6 +163,26 @@
extra_environ=self.extra_environ)
assert_equal(res.body, '{"id": ["The input field id was not expected."]}')
+ def test_register_post_indexerror(self):
+ """
+ Test that we can't add a package if Solr is down.
+ """
+ bad_solr_url = 'http://127.0.0.1/badsolrurl'
+ solr_url = search.common.solr_url
+ try:
+ search.common.solr_url = bad_solr_url
+ plugins.load('synchronous_search')
+
+ assert not self.get_package_by_name(self.package_fixture_data['name'])
+ offset = self.package_offset()
+ data = self.dumps(self.package_fixture_data)
+
+ self.post_json(offset, data, status=500, extra_environ=self.extra_environ)
+ self.remove()
+ finally:
+ plugins.unload('synchronous_search')
+ search.common.solr_url = solr_url
+
def test_entity_get_ok(self):
package_refs = [self.anna.name, self.anna.id]
for ref in package_refs:
@@ -469,6 +479,23 @@
self.app.put(package1_offset, package2_data,
status=self.STATUS_400_BAD_REQUEST)
+ def test_entity_update_indexerror(self):
+ """
+ Test that we can't update a package if Solr is down.
+ """
+ bad_solr_url = 'http://127.0.0.1/badsolrurl'
+ solr_url = search.common.solr_url
+ try:
+ search.common.solr_url = bad_solr_url
+ plugins.load('synchronous_search')
+
+ assert_raises(
+ search.SearchIndexError, self.assert_package_update_ok, 'name', 'post'
+ )
+ finally:
+ plugins.unload('synchronous_search')
+ search.common.solr_url = solr_url
+
def test_entity_delete_ok(self):
# create a package with package_fixture_data
if not self.get_package_by_name(self.package_fixture_data['name']):
--- a/ckan/tests/functional/test_package.py Wed Aug 24 15:56:31 2011 +0100
+++ b/ckan/tests/functional/test_package.py Thu Aug 25 15:26:35 2011 +0100
@@ -1084,6 +1084,21 @@
self.offset = url_for(controller='package', action='edit', id='random_name')
self.res = self.app.get(self.offset, status=404)
+ def test_edit_indexerror(self):
+ bad_solr_url = 'http://127.0.0.1/badsolrurl'
+ solr_url = search.common.solr_url
+ search.common.solr_url = bad_solr_url
+ plugins.load('synchronous_search')
+
+ fv = self.res.forms['package-edit']
+ prefix = ''
+ fv['log_message'] = u'Test log message'
+ res = fv.submit('save', status=500)
+ assert 'Unable to update search index' in res, res
+
+ plugins.unload('synchronous_search')
+ search.common.solr_url = solr_url
+
class TestNew(TestPackageForm):
pkg_names = []
@@ -1367,6 +1382,28 @@
assert plugin.calls['create'] == 1, plugin.calls
plugins.unload(plugin)
+ def test_new_indexerror(self):
+ bad_solr_url = 'http://127.0.0.1/badsolrurl'
+ solr_url = search.common.solr_url
+ search.common.solr_url = bad_solr_url
+ plugins.load('synchronous_search')
+ new_package_name = u'new-package-missing-solr'
+
+ offset = url_for(controller='package', action='new')
+ res = self.app.get(offset)
+ fv = res.forms['package-edit']
+ fv['name'] = new_package_name
+
+ # this package shouldn't actually be created but
+ # add it to the list to purge just in case
+ self.pkg_names.append(new_package_name)
+
+ res = fv.submit('save', status=500)
+ assert 'Unable to add package to search index' in res, res
+
+ plugins.unload('synchronous_search')
+ search.common.solr_url = solr_url
+
class TestNewPreview(TestPackageBase):
pkgname = u'testpkg'
pkgtitle = u'mytesttitle'
Repository URL: https://bitbucket.org/okfn/ckan/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list