[ckan-changes] [okfn/ckan] 610810: template helpers object to help mitigate missing h...
GitHub
noreply at github.com
Mon Apr 23 11:11:18 UTC 2012
Branch: refs/heads/master
Home: https://github.com/okfn/ckan
Commit: 6108109fb6b31062988c6ceeac4701ca71a66626
https://github.com/okfn/ckan/commit/6108109fb6b31062988c6ceeac4701ca71a66626
Author: Toby <toby.junk at gmail.com>
Date: 2012-04-20 (Fri, 20 Apr 2012)
Changed paths:
M ckan/config/environment.py
M ckan/lib/helpers.py
Log Message:
-----------
template helpers object to help mitigate missing helper functions
diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index fa91541..dd15fd0 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -9,7 +9,6 @@
# Suppress benign warning 'Unbuilt egg for setuptools'
warnings.simplefilter('ignore', UserWarning)
-
import pylons
import sqlalchemy
@@ -17,7 +16,6 @@
from pylons.i18n.translation import ugettext
from genshi.template import TemplateLoader
from genshi.filters.i18n import Translator
-from paste.deploy.converters import asbool
import ckan.lib.app_globals as app_globals
import ckan.lib.helpers as h
@@ -27,6 +25,51 @@
+class _Helpers(object):
+ ''' Helper object giving access to template helpers stopping
+ missing functions from causing template exceptions. Useful if
+ templates have helper functions provided by extensions that have
+ not been enabled. '''
+ def __init__(self, helpers, restrict=True):
+ functions = {}
+ allowed = helpers.__allowed_functions__
+
+ for helper in dir(helpers):
+ if restrict and (helper not in allowed):
+ continue
+ functions[helper] = getattr(helpers, helper)
+ self.functions = functions
+
+ # extend helper functions with ones supplied by plugins
+ from ckan.plugins import PluginImplementations
+ from ckan.plugins.interfaces import ITemplateHelpers
+
+ extra_helpers = []
+ for plugin in PluginImplementations(ITemplateHelpers):
+ helpers = plugin.get_helpers()
+ for helper in helpers:
+ if helper in extra_helpers:
+ raise Exception('overwritting extra helper %s' % helper)
+ extra_helpers.append(helper)
+ functions[helper] = helpers[helper]
+
+ @classmethod
+ def null_function(cls, *args, **kw):
+ ''' This function is returned if no helper is found. The idea is
+ to try to allow templates to be rendered even if helpers are
+ missing. Returning the empty string seems to work well.'''
+ return ''
+
+ def __getattr__(self, name):
+ ''' return the function/object requested '''
+ if name in self.functions:
+ return self.functions[name]
+ else:
+ log = logging.getLogger('ckan.helpers')
+ log.critical('Helper function `%s` could not be found (missing extension?)' % name)
+ return self.null_function
+
+
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
@@ -98,25 +141,11 @@ def find_controller(self, controller):
config['routes.map'] = make_map()
config['pylons.app_globals'] = app_globals.Globals()
- if asbool(config.get('ckan.restrict_template_vars', 'false')):
- import ckan.lib.helpers_clean
- config['pylons.h'] = ckan.lib.helpers_clean
- else:
- config['pylons.h'] = h
-
- # extend helper functions with ones supplied by plugins
- from ckan.plugins import PluginImplementations
- from ckan.plugins.interfaces import ITemplateHelpers
-
- extra_helpers = []
- for plugin in PluginImplementations(ITemplateHelpers):
- helpers = plugin.get_helpers()
- for helper in helpers:
- if helper in extra_helpers:
- raise Exception('overwritting extra helper %s' % helper)
- extra_helpers.append(helper)
- setattr(config['pylons.h'], helper, helpers[helper])
+ # add helper functions
+ restrict_helpers = asbool(config.get('ckan.restrict_template_vars', 'false'))
+ helpers = _Helpers(h, restrict_helpers)
+ config['pylons.h'] = helpers
## redo template setup to use genshi.search_path (so remove std template setup)
template_paths = [paths['templates'][0]]
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index b4dd06e..ac1d053 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -707,3 +707,67 @@ def process_names(items):
item = converter(obj, context)
items.append(item)
return items
+
+
+# these are the functions that will end up in `h` template helpers
+# if config option restrict_template_vars is true
+__allowed_functions__ = [
+ # functions defined in ckan.lib.helpers
+ 'redirect_to',
+ 'url',
+ 'url_for',
+ 'url_for_static',
+ 'lang',
+ 'flash',
+ 'flash_error',
+ 'flash_notice',
+ 'flash_success',
+ 'nav_link',
+ 'nav_named_link',
+ 'subnav_link',
+ 'subnav_named_route',
+ 'default_group_type',
+ 'facet_items',
+ 'facet_title',
+ # am_authorized, # depreciated
+ 'check_access',
+ 'linked_user',
+ 'linked_authorization_group',
+ 'group_name_to_title',
+ 'markdown_extract',
+ 'icon',
+ 'icon_html',
+ 'icon_url',
+ 'resource_icon',
+ 'format_icon',
+ 'linked_gravatar',
+ 'gravatar',
+ 'pager_url',
+ 'render_datetime',
+ 'date_str_to_datetime',
+ 'datetime_to_date_str',
+ 'parse_rfc_2822_date',
+ 'time_ago_in_words_from_str',
+ 'button_attr',
+ 'dataset_display_name',
+ 'dataset_link',
+ 'resource_display_name',
+ 'resource_link',
+ 'tag_link',
+ 'group_link',
+ 'dump_json',
+ 'auto_log_message',
+ 'snippet',
+ 'convert_to_dict',
+ 'activity_div',
+ # imported into ckan.lib.helpers
+ 'literal',
+ 'link_to',
+ 'get_available_locales',
+ 'get_locales_dict',
+ 'truncate',
+ 'file',
+ 'mail_to',
+ 'radio',
+ 'submit',
+]
================================================================
Commit: fec1521ed63d4631cfdf40aa3c4232c45f81fcaf
https://github.com/okfn/ckan/commit/fec1521ed63d4631cfdf40aa3c4232c45f81fcaf
Author: Toby <toby.junk at gmail.com>
Date: 2012-04-20 (Fri, 20 Apr 2012)
Changed paths:
R ckan/lib/helpers_clean.py
Log Message:
-----------
lib/helpers_clean.py no longer needed
diff --git a/ckan/lib/helpers_clean.py b/ckan/lib/helpers_clean.py
deleted file mode 100644
index 54c50c8..0000000
--- a/ckan/lib/helpers_clean.py
+++ /dev/null
@@ -1,119 +0,0 @@
-''' This file is part of a plan to clean up the ckan.lib.helpers whilst
-keeping existing versions of ckan from breaking.
-
-
-When it is decided that we will make the template var cleanup
-permanent we will need to implement this using __all__ = [...] in
-lib.helpers itself. Unused imports can also be removed at this time.
-
-yes, yes `from ... import ...` is the work of Satan but this is just a
-short term botch :)
-
-'''
-
-
-from ckan.lib.helpers import (
- # functions defined in ckan.lib.helpers
- redirect_to,
- url,
- url_for,
- url_for_static,
- lang,
- flash,
- flash_error,
- flash_notice,
- flash_success,
- nav_link,
- nav_named_link,
- subnav_link,
- subnav_named_route,
- default_group_type,
- facet_items,
- facet_title,
- # am_authorized, # depreciated
- check_access,
- linked_user,
- linked_authorization_group,
- group_name_to_title,
- markdown_extract,
- icon,
- icon_html,
- icon_url,
- resource_icon,
- format_icon,
- linked_gravatar,
- gravatar,
- pager_url,
- render_datetime,
- date_str_to_datetime,
- datetime_to_date_str,
- parse_rfc_2822_date,
- time_ago_in_words_from_str,
- button_attr,
- dataset_display_name,
- dataset_link,
- resource_display_name,
- resource_link,
- tag_link,
- group_link,
- dump_json,
- auto_log_message,
- snippet,
- convert_to_dict,
- activity_div,
- # imported into ckan.lib.helpers
- literal,
- link_to,
- get_available_locales,
- get_locales_dict,
- truncate,
- file,
- mail_to,
- radio,
- submit,
-)
-
-
-# these are potentially used by templates but hopefully are not
-imported_functions = [
- 'are_there_flash_messages',
- 'auto_discovery_link',
- # 'beaker_cache',
- 'checkbox',
- # 'ckan',
- # 'config',
- 'convert_boolean_attrs',
- 'css_classes',
- 'date',
- 'datetime',
- 'email',
- 'end_form',
- 'escape',
- 'form',
- 'fromstring',
- 'hidden',
- 'i18n',
- 'image',
- 'javascript_link',
- 'json',
- 'link_to_if',
- 'link_to_unless',
- 'markdown',
- 'ol',
- 'paginate',
- 'password',
- # 're',
- 'request',
- 'required_legend',
- 'select',
- 'stylesheet_link',
- 'text',
- 'textarea',
- 'th_sortable',
- 'title',
- 'ul',
- 'url_escape',
- 'urllib',
- 'xml_declaration',
-]
-
================================================================
Commit: 8b31ea1bbcc1ffe852f315303f12d10f48818b56
https://github.com/okfn/ckan/commit/8b31ea1bbcc1ffe852f315303f12d10f48818b56
Author: Toby <toby.junk at gmail.com>
Date: 2012-04-20 (Fri, 20 Apr 2012)
Changed paths:
M ckan/config/environment.py
Log Message:
-----------
clean imports in config/environment.py
diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index dd15fd0..ff3bc45 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -1,29 +1,26 @@
"""Pylons environment configuration"""
import os
-from urlparse import urlparse
import logging
import warnings
-
-from paste.deploy.converters import asbool
-
-# Suppress benign warning 'Unbuilt egg for setuptools'
-warnings.simplefilter('ignore', UserWarning)
+from urlparse import urlparse
import pylons
+from paste.deploy.converters import asbool
import sqlalchemy
-
from pylons import config
-from pylons.i18n.translation import ugettext
from genshi.template import TemplateLoader
from genshi.filters.i18n import Translator
-import ckan.lib.app_globals as app_globals
+import ckan.config.routing as routing
+import ckan.model as model
+import ckan.plugins as p
import ckan.lib.helpers as h
-from ckan.config.routing import make_map
-from ckan import model
-from ckan import plugins
+import ckan.lib.search as search
+import ckan.lib.app_globals as app_globals
+# Suppress benign warning 'Unbuilt egg for setuptools'
+warnings.simplefilter('ignore', UserWarning)
class _Helpers(object):
''' Helper object giving access to template helpers stopping
@@ -41,11 +38,8 @@ def __init__(self, helpers, restrict=True):
self.functions = functions
# extend helper functions with ones supplied by plugins
- from ckan.plugins import PluginImplementations
- from ckan.plugins.interfaces import ITemplateHelpers
-
extra_helpers = []
- for plugin in PluginImplementations(ITemplateHelpers):
+ for plugin in p.PluginImplementations(p.ITemplateHelpers):
helpers = plugin.get_helpers()
for helper in helpers:
if helper in extra_helpers:
@@ -108,12 +102,9 @@ def find_controller(self, controller):
config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# load all CKAN plugins
- plugins.load_all(config)
+ p.load_all(config)
- from ckan.plugins import PluginImplementations
- from ckan.plugins.interfaces import IConfigurer
-
- for plugin in PluginImplementations(IConfigurer):
+ for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
@@ -133,13 +124,13 @@ def find_controller(self, controller):
config['ckan.site_id'] = ckan_host
# Init SOLR settings and check if the schema is compatible
- from ckan.lib.search import SolrSettings, check_solr_schema_version
- SolrSettings.init(config.get('solr_url'),
- config.get('solr_user'),
- config.get('solr_password'))
- check_solr_schema_version()
+ #from ckan.lib.search import SolrSettings, check_solr_schema_version
+ search.SolrSettings.init(config.get('solr_url'),
+ config.get('solr_user'),
+ config.get('solr_password'))
+ search.check_solr_schema_version()
- config['routes.map'] = make_map()
+ config['routes.map'] = routing.make_map()
config['pylons.app_globals'] = app_globals.Globals()
# add helper functions
@@ -187,9 +178,6 @@ def template_loaded(template):
if not model.meta.engine:
model.init_model(engine)
- from ckan.plugins import PluginImplementations
- from ckan.plugins.interfaces import IConfigurable
-
- for plugin in PluginImplementations(IConfigurable):
+ for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
================================================================
Commit: fbf7754a4b5e4054ea67307014eee6cbd9510029
https://github.com/okfn/ckan/commit/fbf7754a4b5e4054ea67307014eee6cbd9510029
Author: Toby <toby.junk at gmail.com>
Date: 2012-04-23 (Mon, 23 Apr 2012)
Changed paths:
M ckan/ckan_nose_plugin.py
M ckan/config/environment.py
M ckan/controllers/group.py
M ckan/controllers/package.py
M ckan/controllers/tag.py
M ckan/controllers/user.py
M ckan/lib/base.py
M ckan/lib/celery_app.py
M ckan/lib/create_test_data.py
M ckan/lib/dictization/model_dictize.py
M ckan/lib/helpers.py
M ckan/logic/__init__.py
M ckan/logic/action/get.py
M ckan/logic/auth/get.py
M ckan/logic/auth/publisher/get.py
M ckan/logic/schema.py
M ckan/model/__init__.py
M ckan/plugins/interfaces.py
M ckan/public/scripts/application.js
M ckan/templates/_snippet/data-api-help.html
M ckan/templates/_util.html
M ckan/templates/facets.html
M ckan/templates/group/read.html
M ckan/templates/package/search.html
M ckan/templates/tag/index.html
M ckan/templates/tag/read.html
M ckan/tests/lib/test_dictization.py
A ckan/tests/logic/__init__.py
M ckan/tests/logic/test_action.py
A ckanext/multilingual/plugin.py
A ckanext/multilingual/solr/dutch_stop.txt
A ckanext/multilingual/solr/english_stop.txt
A ckanext/multilingual/solr/fr_elision.txt
A ckanext/multilingual/solr/french_stop.txt
A ckanext/multilingual/solr/german_stop.txt
A ckanext/multilingual/solr/greek_stopwords.txt
A ckanext/multilingual/solr/italian_stop.txt
A ckanext/multilingual/solr/polish_stop.txt
A ckanext/multilingual/solr/portuguese_stop.txt
A ckanext/multilingual/solr/romanian_stop.txt
A ckanext/multilingual/solr/schema.xml
A ckanext/multilingual/solr/spanish_stop.txt
A ckanext/multilingual/tests/test_multilingual_plugin.py
M doc/_themes/sphinx-theme-okfn
M doc/api.rst
M doc/apiv3.rst
M doc/datastore.rst
M doc/i18n.rst
M doc/index.rst
A doc/multilingual.rst
M doc/paster.rst
M doc/post-installation.rst
A doc/using-data-api.rst
M setup.py
Log Message:
-----------
Merge branch 'master' of github.com:okfn/ckan
Conflicts:
ckan/lib/helpers_clean.py
- deleted as no longer used to manage template helpers
diff --git a/ckan/ckan_nose_plugin.py b/ckan/ckan_nose_plugin.py
index 7018e31..bc995db 100644
--- a/ckan/ckan_nose_plugin.py
+++ b/ckan/ckan_nose_plugin.py
@@ -30,7 +30,6 @@ def startContext(self, ctx):
# init_db is run at the start of every class because
# when you use an in-memory sqlite db, it appears that
# the db is destroyed after every test when you Session.Remove().
- model.repo.init_db()
## This is to make sure the configuration is run again.
## Plugins use configure to make their own tables and they
@@ -40,6 +39,7 @@ def startContext(self, ctx):
for plugin in PluginImplementations(IConfigurable):
plugin.configure(config)
+ model.repo.init_db()
def options(self, parser, env):
parser.add_option(
diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index ff3bc45..580e472 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -171,9 +171,8 @@ def template_loaded(template):
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
- engine = sqlalchemy.create_engine(ckan_db)
- else:
- engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
+ config['sqlalchemy.url'] = ckan_db
+ engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
if not model.meta.engine:
model.init_model(engine)
diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index 4e7cda3..95a4597 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -72,7 +72,7 @@ def index(self):
group_type = self._guess_group_type()
context = {'model': model, 'session': model.Session,
- 'user': c.user or c.author}
+ 'user': c.user or c.author, 'for_view': True}
data_dict = {'all_fields': True}
@@ -194,6 +194,7 @@ def pager_url(q=None, page=None):
items_per_page=limit
)
c.facets = query['facets']
+ c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Group search error: %r', se.args)
diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py
index e90d658..18c2b2a 100644
--- a/ckan/controllers/package.py
+++ b/ckan/controllers/package.py
@@ -210,6 +210,7 @@ def pager_url(q=None, page=None):
items_per_page=limit
)
c.facets = query['facets']
+ c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
diff --git a/ckan/controllers/tag.py b/ckan/controllers/tag.py
index f623f05..06bccd7 100644
--- a/ckan/controllers/tag.py
+++ b/ckan/controllers/tag.py
@@ -25,9 +25,9 @@ def index(self):
c.q = request.params.get('q', '')
context = {'model': model, 'session': model.Session,
- 'user': c.user or c.author}
+ 'user': c.user or c.author, 'for_view': True}
- data_dict = {}
+ data_dict = {'all_fields': True}
if c.q:
page = int(request.params.get('page', 1))
@@ -58,7 +58,7 @@ def index(self):
def read(self, id):
context = {'model': model, 'session': model.Session,
- 'user': c.user or c.author}
+ 'user': c.user or c.author, 'for_view': True}
data_dict = {'id':id}
try:
diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py
index 0a26d59..7f46b4f 100644
--- a/ckan/controllers/user.py
+++ b/ckan/controllers/user.py
@@ -83,7 +83,7 @@ def index(self):
def read(self, id=None):
context = {'model': model,
- 'user': c.user or c.author}
+ 'user': c.user or c.author, 'for_view': True}
data_dict = {'id':id,
'user_obj':c.userobj}
try:
diff --git a/ckan/lib/base.py b/ckan/lib/base.py
index 075029c..dd1f03b 100644
--- a/ckan/lib/base.py
+++ b/ckan/lib/base.py
@@ -245,8 +245,8 @@ def __after__(self, action, **params):
def _set_cors(self):
response.headers['Access-Control-Allow-Origin'] = "*"
- response.headers['Access-Control-Allow-Methods'] = "POST, PUT, GET, DELETE"
- response.headers['Access-Control-Allow-Headers'] = "X-CKAN-API-KEY, Content-Type"
+ response.headers['Access-Control-Allow-Methods'] = "POST, PUT, GET, DELETE, OPTIONS"
+ response.headers['Access-Control-Allow-Headers'] = "X-CKAN-API-KEY, Authorization, Content-Type"
def _get_user(self, reference):
return model.User.by_name(reference)
diff --git a/ckan/lib/celery_app.py b/ckan/lib/celery_app.py
index cd87aa7..27d2951 100644
--- a/ckan/lib/celery_app.py
+++ b/ckan/lib/celery_app.py
@@ -1,5 +1,6 @@
import ConfigParser
import os
+from pylons import config as pylons_config
from pkg_resources import iter_entry_points
#from celery.loaders.base import BaseLoader
@@ -12,16 +13,22 @@
config = ConfigParser.ConfigParser()
config_file = os.environ.get('CKAN_CONFIG')
+
if not config_file:
config_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)), '../../development.ini')
config.read(config_file)
+sqlalchemy_url = pylons_config.get('sqlalchemy.url')
+if not sqlalchemy_url:
+ sqlalchemy_url = config.get('app:main', 'sqlalchemy.url')
+
+
default_config = dict(
BROKER_BACKEND = 'sqlalchemy',
- BROKER_HOST = config.get('app:main', 'sqlalchemy.url'),
- CELERY_RESULT_DBURI = config.get('app:main', 'sqlalchemy.url'),
+ BROKER_HOST = sqlalchemy_url,
+ CELERY_RESULT_DBURI = sqlalchemy_url,
CELERY_RESULT_BACKEND = 'database',
CELERY_RESULT_SERIALIZER = 'json',
CELERY_TASK_SERIALIZER = 'json',
diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py
index 18e81be..44d5c64 100644
--- a/ckan/lib/create_test_data.py
+++ b/ckan/lib/create_test_data.py
@@ -6,13 +6,16 @@ class CreateTestData(cli.CkanCommand):
'''Create test data in the database.
Tests can also delete the created objects easily with the delete() method.
- create-test-data - annakarenina and warandpeace
- create-test-data search - realistic data to test search
- create-test-data gov - government style data
- create-test-data family - package relationships data
- create-test-data user - create a user 'tester' with api key 'tester'
+ create-test-data - annakarenina and warandpeace
+ create-test-data search - realistic data to test search
+ create-test-data gov - government style data
+ create-test-data family - package relationships data
+ create-test-data user - create a user 'tester' with api key 'tester'
+ create-test-data translations - annakarenina, warandpeace, and some test
+ translations of terms
create-test-data vocabs - annakerenina, warandpeace, and some test
vocabularies
+
'''
summary = __doc__.split('\n')[0]
usage = __doc__
@@ -53,6 +56,8 @@ def command(self):
self.create_gov_test_data()
elif cmd == 'family':
self.create_family_test_data()
+ elif cmd == 'translations':
+ self.create_translations_test_data()
elif cmd == 'vocabs':
self.create_vocabs_test_data()
else:
@@ -92,6 +97,44 @@ def create_test_user(cls):
cls.user_refs.append(u'tester')
@classmethod
+
+ def create_translations_test_data(cls):
+ import ckan.model
+ CreateTestData.create()
+ rev = ckan.model.repo.new_revision()
+ rev.author = CreateTestData.author
+ rev.message = u'Creating test translations.'
+
+ sysadmin_user = ckan.model.User.get('testsysadmin')
+ package = ckan.model.Package.get('annakarenina')
+
+ # Add some new tags to the package.
+ # These tags are codes that are meant to be always translated before
+ # display, if not into the user's current language then into the
+ # fallback language.
+ package.add_tags([ckan.model.Tag('123'), ckan.model.Tag('456'),
+ ckan.model.Tag('789')])
+
+ # Add the above translations to CKAN.
+ for (lang_code, translations) in (('de', german_translations),
+ ('fr', french_translations), ('en', english_translations)):
+ for term in terms:
+ if term in translations:
+ data_dict = {
+ 'term': term,
+ 'term_translation': translations[term],
+ 'lang_code': lang_code,
+ }
+ context = {
+ 'model': ckan.model,
+ 'session': ckan.model.Session,
+ 'user': sysadmin_user.name,
+ }
+ ckan.logic.action.update.term_translation_update(context,
+ data_dict)
+
+ ckan.model.Session.commit()
+
def create_vocabs_test_data(cls):
import ckan.model
CreateTestData.create()
@@ -770,3 +813,49 @@ def get_all_data(cls):
}
}
]
+
+# Some test terms and translations.
+terms = ('A Novel By Tolstoy',
+ 'Index of the novel',
+ 'russian',
+ 'tolstoy',
+ "Dave's books",
+ "Roger's books",
+ 'Other (Open)',
+ 'romantic novel',
+ 'book',
+ '123',
+ '456',
+ '789',
+ 'plain text',
+ 'Roger likes these books.',
+)
+english_translations = {
+ '123': 'jealousy',
+ '456': 'realism',
+ '789': 'hypocrisy',
+}
+german_translations = {
+ 'A Novel By Tolstoy': 'Roman von Tolstoi',
+ 'Index of the novel': 'Index des Romans',
+ 'russian': 'Russisch',
+ 'tolstoy': 'Tolstoi',
+ "Dave's books": 'Daves Bucher',
+ "Roger's books": 'Rogers Bucher',
+ 'Other (Open)': 'Andere (Open)',
+ 'romantic novel': 'Liebesroman',
+ 'book': 'Buch',
+ '456': 'Realismus',
+ '789': 'Heuchelei',
+ 'plain text': 'Klartext',
+ 'Roger likes these books.': 'Roger mag diese Bucher.'
+}
+french_translations = {
+ 'A Novel By Tolstoy': 'A Novel par Tolstoi',
+ 'Index of the novel': 'Indice du roman',
+ 'russian': 'russe',
+ 'romantic novel': 'roman romantique',
+ 'book': 'livre',
+ '123': 'jalousie',
+ '789': 'hypocrisie',
+}
diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py
index d0e7d6c..e3bbd1c 100644
--- a/ckan/lib/dictization/model_dictize.py
+++ b/ckan/lib/dictization/model_dictize.py
@@ -1,7 +1,7 @@
import datetime
from pylons import config
from sqlalchemy.sql import select
-
+import datetime
import ckan.model
import ckan.misc
import ckan.logic as logic
@@ -32,6 +32,11 @@ def group_list_dictize(obj_list, context,
group_dict['packages'] = len(obj.active_packages().all())
+ if context.get('for_view'):
+ for item in plugins.PluginImplementations(
+ plugins.IGroupController):
+ group_dict = item.before_view(group_dict)
+
result_list.append(group_dict)
return sorted(result_list, key=sort_key, reverse=reverse)
@@ -153,6 +158,7 @@ def package_dictize(pkg, context):
q = q.where(resource_group.c.package_id == pkg.id)
result = _execute_with_revision(q, res_rev, context)
result_dict["resources"] = resource_list_dictize(result, context)
+
#tags
tag_rev = model.package_tag_revision_table
tag = model.tag_table
@@ -161,6 +167,14 @@ def package_dictize(pkg, context):
).where(tag_rev.c.package_id == pkg.id)
result = _execute_with_revision(q, tag_rev, context)
result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"])
+
+ # Add display_names to tags. At first a tag's display_name is just the
+ # same as its name, but the display_name might get changed later (e.g.
+ # translated into another language by the multilingual extension).
+ for tag in result_dict['tags']:
+ assert not tag.has_key('display_name')
+ tag['display_name'] = tag['name']
+
#extras
extra_rev = model.extra_revision_table
q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id)
@@ -210,10 +224,9 @@ def package_dictize(pkg, context):
if pkg.metadata_created else None
if context.get('for_view'):
- for item in plugins.PluginImplementations(plugins.IPackageController):
+ for item in plugins.PluginImplementations( plugins.IPackageController):
result_dict = item.before_view(result_dict)
-
return result_dict
def _get_members(context, group, member_type):
@@ -226,7 +239,6 @@ def _get_members(context, group, member_type):
filter(model.Member.state == 'active').\
filter(model.Member.table_name == member_type[:-1]).all()
-
def group_dictize(group, context):
model = context['model']
result_dict = d.table_dictize(group, context)
@@ -279,6 +291,18 @@ def tag_dictize(tag, context):
result_dict = d.table_dictize(tag, context)
result_dict["packages"] = d.obj_list_dictize(tag.packages, context)
+
+ # Add display_names to tags. At first a tag's display_name is just the
+ # same as its name, but the display_name might get changed later (e.g.
+ # translated into another language by the multilingual extension).
+ assert not result_dict.has_key('display_name')
+ result_dict['display_name'] = result_dict['name']
+
+ if context.get('for_view'):
+ for item in plugins.PluginImplementations(
+ plugins.ITagController):
+ result_dict = item.before_view(result_dict)
+
return result_dict
def user_list_dictize(obj_list, context,
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index ac1d053..dfdc0a8 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -7,6 +7,7 @@
"""
import email.utils
import datetime
+import logging
import re
import urllib
@@ -45,6 +46,8 @@
except ImportError:
import simplejson as json
+_log = logging.getLogger(__name__)
+
def redirect_to(*args, **kw):
'''A routes.redirect_to wrapper to retain the i18n settings'''
kw['__ckan_no_root'] = True
@@ -331,8 +334,41 @@ def _subnav_named_route(text, routename, **kwargs):
def default_group_type():
return str( config.get('ckan.default.group_type', 'group') )
+def unselected_facet_items(facet, limit=10):
+ '''Return the list of unselected facet items for the given facet, sorted
+ by count.
+
+ Returns the list of unselected facet contraints or facet items (e.g. tag
+ names like "russian" or "tolstoy") for the given search facet (e.g.
+ "tags"), sorted by facet item count (i.e. the number of search results that
+ match each facet item).
+
+ Reads the complete list of facet items for the given facet from
+ c.search_facets, and filters out the facet items that the user has already
+ selected.
+
+ Arguments:
+ facet -- the name of the facet to filter.
+ limit -- the max. number of facet items to return.
+
+ '''
+ if not c.search_facets or \
+ not c.search_facets.get(facet) or \
+ not c.search_facets.get(facet).get('items'):
+ return []
+ facets = []
+ for facet_item in c.search_facets.get(facet)['items']:
+ if not len(facet_item['name'].strip()):
+ continue
+ if not (facet, facet_item['name']) in request.params.items():
+ facets.append(facet_item)
+ return sorted(facets, key=lambda item: item['count'], reverse=True)[:limit]
def facet_items(*args, **kwargs):
+ """
+ DEPRECATED: Use the new facet data structure, and `unselected_facet_items()`
+ """
+ _log.warning('Deprecated function: ckan.lib.helpers:facet_items(). Will be removed in v1.8')
# facet_items() used to need c passing as the first arg
# this is depriciated as pointless
# throws error if ckan.restrict_template_vars is True
diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py
index 513f291..30cd4ae 100644
--- a/ckan/logic/__init__.py
+++ b/ckan/logic/__init__.py
@@ -39,6 +39,7 @@ class NotAuthorized(ActionError):
class ParameterError(ActionError):
pass
+
class ValidationError(ParameterError):
def __init__(self, error_dict, error_summary=None, extra_msg=None):
self.error_dict = error_dict
@@ -224,3 +225,29 @@ def get_action(action):
_actions.update(fetched_actions)
return _actions.get(action)
+def get_or_bust(data_dict, keys):
+ '''Try and get values from dictionary and if they are not there
+ raise a validataion error.
+
+ data_dict: a dictionary
+ keys: either a single string key in which case will return a single value,
+ or a iterable which will return a tuple for unpacking purposes.
+
+ e.g single_value = get_or_bust(data_dict, 'a_key')
+ value_1, value_2 = get_or_bust(data_dict, ['key1', 'key2'])
+ '''
+ values = []
+ errors = {}
+
+ if isinstance(keys, basestring):
+ keys = [keys]
+ for key in keys:
+ value = data_dict.get(key)
+ if not value:
+ errors[key] = _('Missing value')
+ values.append(value)
+ if errors:
+ raise ValidationError(errors)
+ if len(values) == 1:
+ return values[0]
+ return tuple(values)
diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
index 21429d3..b50d53c 100644
--- a/ckan/logic/action/get.py
+++ b/ckan/logic/action/get.py
@@ -6,7 +6,7 @@
import webhelpers.html
from sqlalchemy.sql import select
from sqlalchemy.orm import aliased
-from sqlalchemy import or_, and_, func, desc, case
+from sqlalchemy import or_, and_, func, desc, case, text
import ckan
import ckan.authz
@@ -31,6 +31,7 @@
check_access = logic.check_access
NotFound = logic.NotFound
ValidationError = logic.ValidationError
+get_or_bust = logic.get_or_bust
def _package_list_with_resources(context, package_revision_list):
package_list = []
@@ -417,9 +418,27 @@ def resource_show(context, data_dict):
raise NotFound
check_access('resource_show', context, data_dict)
-
return model_dictize.resource_dictize(resource, context)
+def resource_status_show(context, data_dict):
+
+ model = context['model']
+ id = get_or_bust(data_dict, 'id')
+
+ check_access('resource_status_show', context, data_dict)
+
+ # needs to be text query as celery tables are not in our model
+ q = text("""select status, date_done, traceback, task_status.*
+ from task_status left join celery_taskmeta
+ on task_status.value = celery_taskmeta.task_id and key = 'celery_task_id'
+ where entity_id = :entity_id """)
+
+ result = model.Session.connection().execute(q, entity_id=id)
+ result_list = [table_dictize(row, context) for row in result]
+
+ return result_list
+
+
def revision_show(context, data_dict):
model = context['model']
api = context.get('api_version')
@@ -753,12 +772,40 @@ def package_search(context, data_dict):
'results': results
}
+ # Transform facets into a more useful data structure.
+ restructured_facets = {}
+ for key, value in search_results['facets'].items():
+ restructured_facets[key] = {
+ 'title': key,
+ 'items': []
+ }
+ for key_, value_ in value.items():
+ new_facet_dict = {}
+ new_facet_dict['name'] = key_
+ if key == 'groups':
+ group = model.Group.get(key_)
+ if group:
+ new_facet_dict['display_name'] = group.display_name
+ else:
+ new_facet_dict['display_name'] = key_
+ else:
+ new_facet_dict['display_name'] = key_
+ new_facet_dict['count'] = value_
+ restructured_facets[key]['items'].append(new_facet_dict)
+ search_results['search_facets'] = restructured_facets
+
# check if some extension needs to modify the search results
for item in plugins.PluginImplementations(plugins.IPackageController):
search_results = item.after_search(search_results,data_dict)
- return search_results
+ # After extensions have had a chance to modify the facets, sort them by
+ # display name.
+ for facet in search_results['search_facets']:
+ search_results['search_facets'][facet]['items'] = sorted(
+ search_results['search_facets'][facet]['items'],
+ key=lambda facet: facet['display_name'], reverse=True)
+ return search_results
def resource_search(context, data_dict):
model = context['model']
@@ -933,13 +980,13 @@ def term_translation_show(context, data_dict):
q = select([trans_table])
- if 'term' not in data_dict:
- raise ValidationError({'term': 'term not in data'})
+ if 'terms' not in data_dict:
+ raise ValidationError({'terms': 'terms not in data'})
- q = q.where(trans_table.c.term == data_dict['term'])
+ q = q.where(trans_table.c.term.in_(data_dict['terms']))
- if 'lang_code' in data_dict:
- q = q.where(trans_table.c.lang_code == data_dict['lang_code'])
+ if 'lang_codes' in data_dict:
+ q = q.where(trans_table.c.lang_code.in_(data_dict['lang_codes']))
conn = model.Session.connection()
cursor = conn.execute(q)
diff --git a/ckan/logic/auth/get.py b/ckan/logic/auth/get.py
index 27e8436..9a04520 100644
--- a/ckan/logic/auth/get.py
+++ b/ckan/logic/auth/get.py
@@ -154,6 +154,9 @@ def format_autocomplete(context, data_dict):
def task_status_show(context, data_dict):
return {'success': True}
+def resource_status_show(context, data_dict):
+ return {'success': True}
+
## Modifications for rest api
def package_show_rest(context, data_dict):
diff --git a/ckan/logic/auth/publisher/get.py b/ckan/logic/auth/publisher/get.py
index b2dbd9a..f640625 100644
--- a/ckan/logic/auth/publisher/get.py
+++ b/ckan/logic/auth/publisher/get.py
@@ -160,6 +160,9 @@ def format_autocomplete(context, data_dict):
def task_status_show(context, data_dict):
return {'success': True}
+def resource_status_show(context, data_dict):
+ return {'success': True}
+
## Modifications for rest api
def package_show_rest(context, data_dict):
diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py
index b07d2bb..0d41722 100644
--- a/ckan/logic/schema.py
+++ b/ckan/logic/schema.py
@@ -85,6 +85,7 @@ def default_tags_schema():
'vocabulary_id': [ignore_missing, unicode, vocabulary_id_exists],
'revision_timestamp': [ignore],
'state': [ignore],
+ 'display_name': [ignore],
}
return schema
diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py
index df0d2f5..f254318 100644
--- a/ckan/model/__init__.py
+++ b/ckan/model/__init__.py
@@ -79,6 +79,16 @@ def init_db(self):
else:
if not self.tables_created_and_initialised:
self.upgrade_db()
+ ## make sure celery tables are made as celery only makes them after
+ ## adding a task
+ try:
+ import ckan.lib.celery_app as celery_app
+ backend = celery_app.celery.backend
+ ##This creates the database tables as a side effect, can not see another way
+ ##to make tables unless you actually create a task.
+ celery_result_session = backend.ResultSession()
+ except ImportError:
+ pass
self.init_configuration_data()
self.tables_created_and_initialised = True
diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py
index ee092d7..bc2eb88 100644
--- a/ckan/plugins/interfaces.py
+++ b/ckan/plugins/interfaces.py
@@ -13,7 +13,9 @@
'IPackageController', 'IPluginObserver',
'IConfigurable', 'IConfigurer', 'IAuthorizer',
'IActions', 'IResourceUrlChange', 'IDatasetForm',
- 'IGroupForm', 'ITemplateHelpers',
+ 'IGroupForm',
+ 'ITagController',
+ 'ITemplateHelpers',
]
from inspect import isclass
@@ -181,6 +183,21 @@ class IResourceUrlChange(Interface):
def notify(self, resource):
pass
+class ITagController(Interface):
+ '''
+ Hook into the Tag controller. These will usually be called just before
+ committing or returning the respective object, i.e. all validation,
+ synchronization and authorization setup are complete.
+
+ '''
+ def before_view(self, tag_dict):
+ '''
+ Extensions will recieve this before the tag gets displayed. The
+ dictionary passed will be the one that gets sent to the template.
+
+ '''
+ return tag_dict
+
class IGroupController(Interface):
"""
Hook into the Group controller. These will
diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js
index 995e703..d94bbd4 100644
--- a/ckan/public/scripts/application.js
+++ b/ckan/public/scripts/application.js
@@ -958,7 +958,6 @@ CKAN.Utils = function($, my) {
, select: function(event, ui) {
var input_box = $(this);
input_box.val('');
- var parent_dd = input_box.parent('dd');
var old_name = input_box.attr('name');
var field_name_regex = /^(\S+)__(\d+)__(\S+)$/;
var split = old_name.match(field_name_regex);
@@ -967,10 +966,15 @@ CKAN.Utils = function($, my) {
input_box.attr('name', new_name);
input_box.attr('id', new_name);
-
- parent_dd.before(
- '<input type="hidden" name="' + old_name + '" value="' + ui.item.value + '">' + '<dd>' + ui.item.label + '</dd>'
- );
+
+ var $new = $('<div class="ckan-dataset-to-add"><p></p></div>');
+ $new.append($('<input type="hidden" />').attr('name', old_name).val(ui.item.value));
+ $new.append('<i class="icon-plus-sign"></i> ');
+ $new.append(ui.item.label);
+ input_box.after($new);
+
+ // prevent setting value in autocomplete box
+ return false;
}
});
};
diff --git a/ckan/templates/_snippet/data-api-help.html b/ckan/templates/_snippet/data-api-help.html
index 5a499c9..bab65b2 100644
--- a/ckan/templates/_snippet/data-api-help.html
+++ b/ckan/templates/_snippet/data-api-help.html
@@ -15,102 +15,51 @@
</h3>
</div>
<div class="modal-body">
- <p><strong>Access resource data via a web API with powerful query
- support</strong>. Further information in the <a
- href="http://docs.ckan.org/en/latest/storage/datastore.html" target="_blank">main
- CKAN Data API and DataStore documentation</a>.</p>
+ <p><strong>Access (and update) resource data via a web API with powerful query
+ support</strong>.</p>
+
+ <ul>
+ <li>Quickstart info below</li>
+ <li><a
+ href="http://docs.ckan.org/en/latest/using-data-api.html" target="_blank">
+ Tutorial, examples and full details in main CKAN documentation</a></li>
+ </ul>
<div class="accordion-group">
<div class="accordion-heading">
- <a class="accordion-toggle" data-toggle="collapse" href=".collapse-endpoints">Endpoints »</a>
- </div>
- <div class="collapse-endpoints in accordion-body collapse">
- <div class="accordion-inner">
- <p>The Data API builds directly on ElasticSearch, with a resource API
- endpoint being equivalent to a single index 'type' in ElasticSearch.
- This means you can directly re-use <a
- href="http://www.elasticsearch.org/guide/appendix/clients.html"
- _target="blank">ElasticSearch
- client libraries</a> when connecting to the API endpoint.</p>
- <table class="table-condensed table-striped table-bordered">
- <thead></thead>
- <tbody>
- <tr>
- <th>Query example</th>
- <td>
- <code><a href="${datastore_api}/_search?size=5&pretty=true" target="_blank">${datastore_api}/_search?size=5&pretty=true</a></code>
- </td>
- </tr>
- <tr>
- <th>Schema (Mapping)</th>
- <td>
- <code><a href="${datastore_api}/_mapping" target="_blank">${datastore_api}/_mapping?pretty=true</a></code>
- </td>
- </tr>
- <tr>
- <th>Base</th>
- <td><code>${datastore_api}</code></td>
- </tr>
- </tbody>
- </table>
- </div>
- </div>
- </div>
-
- <div class="accordion-group">
- <div class="accordion-heading">
- <a class="accordion-toggle" href=".collapse-querying" data-toggle="collapse">Querying
- »</a>
+ <a class="accordion-toggle" data-toggle="collapse" href=".collapse-endpoints">Example URLs and Endpoints»</a>
</div>
- <div class="collapse collapse-querying accordion-body in">
+ <div class="collapse-endpoints in accordion-body">
<div class="accordion-inner">
- <p><strong>Basic queries</strong> can be done using the <code>q</code>
- parameter in the query string which supports the <a
- href="http://lucene.apache.org/core/old_versioned_docs/versions/3_0_0/queryparsersyntax.html"
- target="_blank">Lucene
- query parser syntax</a> and hence filters on specific fields
- (<code>e.g. fieldname:value</code>), wildcards (e.g. <code>abc*</code>)
- and more. Full query parameters and options in the <a
- href="http://www.elasticsearch.org/guide/reference/api/search/uri-request.html"
- target="_blank">ElasticSearch
- URI request docs</a>.</p>
+ <strong>Query example (first 5 results)</strong>
+ <p>
+ <code><a href="${datastore_api}/_search?size=5&pretty=true" target="_blank">${datastore_api}/_search?size=5&pretty=true</a></code>
+ </p>
+
+ <strong>Query example (results with 'jones' in <code>title</code> field)</strong>
+ <p>
+ <code><a href="${datastore_api}/_search?q=title:jones&size=5&pretty=true"
+ target="_blank">${datastore_api}/_search?q=title:jones&size=5&pretty=true</a></code>
+ </p>
- <p><strong>More complex queries</strong>, including those that involve
- faceting and statistical operations, should use the full ElasticSearch
- query language in which the query is a JSON structure sent in the
- <code>?source=</code> parameter. See <a
- href="http://www.elasticsearch.org/guide/reference/api/search/"
- target="_blank">ElasticSearch
- query documentation</a>.</p>
+ <strong>Schema (Mapping)</strong>
+ <p>
+ <code><a href="${datastore_api}/_mapping" target="_blank">${datastore_api}/_mapping?pretty=true</a></code>
+ </p>
- <p>JSONP support is available via a simple callback query parameter:
- <code>?callback=my_callback_name</code>.</p>
+ <strong>Endpoint (for clients)</strong>
+ <p>
+ <code>${datastore_api}</code>
+ </p>
</div>
</div>
</div>
<div class="accordion-group">
<div class="accordion-heading">
- <a class="accordion-toggle" href=".collapse-curl"
- data-toggle="collapse">Example: cURL (or Browser) »</a>
- </div>
- <div class="accordion-body collapse-curl collapse in">
- <div class="accordion-inner">
- <p>The following examples utilize the <a href="http://curl.haxx.se/">cURL</a>
- command line utility. If you prefer, you you can just open the relevant urls in
- your browser.</p>
-
- <pre>
-// added pretty=true to get the json results pretty printed
-curl ${datastore_api}/_search?q=title:jones&size=5&pretty=true</pre>
- </div>
- </div>
- </div>
- <div class="accordion-group">
- <div class="accordion-heading">
<a class="accordion-toggle" href=".collapse-javascript" data-toggle="collapse">Example: Javascript</a>
</div>
- <div class="accordion-body collapse-javascript collapse in">
+ <div class="accordion-body collapse-javascript in">
<div class="accordion-inner">
<p>A simple ajax (JSONP) request to the data API using jQuery.</p>
<pre>
@@ -119,7 +68,8 @@
q: 'title:jones' // query on the title field for 'jones'
};
$.ajax({
- url: ${datastore_api}/_search,
+ url: '${datastore_api}/_search',
+ data: data,
dataType: 'jsonp',
success: function(data) {
alert('Total results found: ' + data.hits.total)
@@ -128,6 +78,22 @@
</div>
</div>
</div>
+
+ <div class="accordion-group">
+ <div class="accordion-heading">
+ <a class="accordion-toggle" href=".collapse-javascript" data-toggle="collapse">Example: Python</a>
+ </div>
+ <div class="accordion-body collapse-javascript in">
+ <div class="accordion-inner">
+ <pre>
+import urllib
+url = '${datastore_api}/_search?size=5&q=title:jones'
+fileobj = urllib.urlopen(url)
+print fileobj.read()
+</pre>
+ </div>
+ </div>
+ </div>
</div>
</div>
diff --git a/ckan/templates/_util.html b/ckan/templates/_util.html
index f470b07..a98504f 100644
--- a/ckan/templates/_util.html
+++ b/ckan/templates/_util.html
@@ -40,19 +40,11 @@
<ul py:def="tag_list(tags)" class="tags clearfix">
<py:for each="tag in tags">
<li py:if="not tag.get('vocabulary_id')">
- ${h.link_to(tag['name'], h.url_for(controller='tag', action='read', id=tag['name']))}
+ ${h.link_to(tag['display_name'], h.url_for(controller='tag', action='read', id=tag['name']))}
</li>
</py:for>
</ul>
- <!--! List of tags: pass in a list of tag name and this renders the standard
- tag listing -->
- <ul py:def="tag_list_from_name(tags)" class="tags clearfix">
- <li py:for="tag in tags">
- ${h.link_to(tag, h.url_for(controller='tag', action='read', id=tag))}
- </li>
- </ul>
-
<!--! List of users: pass in a collection of users and this renders the standard
user listing -->
<ul py:def="user_list(users)" class="users">
diff --git a/ckan/templates/facets.html b/ckan/templates/facets.html
index f50ac3b..f8c104b 100644
--- a/ckan/templates/facets.html
+++ b/ckan/templates/facets.html
@@ -5,43 +5,176 @@
py:strip=""
>
+<!--
+Construct a facet <div> populated with links to filtered results.
+
+name
+ The field name identifying the facet field, eg. "tags"
+
+title
+ The title of the facet, eg. "Tags", or "Tag Cloud"
+
+label_function
+ Renders the human-readable label for each facet value.
+ If defined, this should be a callable that accepts a `facet_item`.
+ eg. lambda facet_item: facet_item.display_name.upper()
+ By default it displays the facet item's display name, which should
+ usually be good enough
+
+if_empty
+ A string, which if defined, and the list of possible facet items is empty,
+ is displayed in lieu of an empty list.
+
+count_label
+ A callable which accepts an integer, and returns a string. This controls
+ how a facet-item's count is displayed.
+
+-->
+<py:def function="facet_div(name, title, limit=10, label_function=lambda item: item.display_name, if_empty=None, count_label=lambda c: ' (%d)'%c)">
+ <div py:if="if_empty is not None or h.unselected_facet_items(name, limit)" class="facet-box">
+ <h2>${h.facet_title(title)}</h2>
+ <ul class="facet-options">
+ ${facet_li(name, limit=limit, label_function=label_function, if_empty=None, count_label=count_label)}
+ </ul>
+ <p py:if="not h.unselected_facet_items(name, limit)">${if_empty}</p>
+ </div>
+</py:def>
+
+<!--
+Generate <li>s for facet items. The generated tags are not wrapped by any
+other tag, ie - it's up to the caller to wrap them in something suitable.
+
+name
+ The field name identifying the facet field, eg. "tags"
+
+label_function
+ Renders the human-readable label for each facet value.
+ If defined, this should be a callable that accepts a `facet_item`.
+ eg. lambda facet_item: facet_item.display_name.upper()
+ By default it displays the facet item's display name, which should
+ usually be good enough
+
+if_empty
+ A string, which if defined, and the list of possible facet items is empty,
+ is displayed in a <li> tag in lieu of an empty list.
+
+count_label
+ A callable which accepts an integer, and returns a string. This controls
+ how a facet-item's count is displayed.
+
+-->
+<py:def function="facet_li(name, limit=5, label_function=lambda item: item.display_name, if_empty=None, count_label=lambda c: ' (%d)'%c)">
+
+ <li py:if="if_empty and not h.unselected_facet_items(name, limit)">${if_empty}</li>
+
+ <li py:for="facet_item in h.unselected_facet_items(name, limit)">
+ <a href="${c.drill_down_url(**{name: facet_item.name})}">
+ ${label_function(facet_item)}
+ </a>
+ ${count_label(facet_item['count'])}
+ </li>
+</py:def>
+
+<!--
+DEPRECATED. Provided only for backward compatibility with existing plugins.
+ Use `facet_div` instead.
+
+Similar to the above, `facet_div` function; this helper creates a <div>
+populated with links to filtered search results.
+
+Differences with the preferred `facet_div` function:
+
+ * `title` argument is a callable
+ * `label` is a callable that accepts a string cf. `label_function` which is
+ a callable that accepts a `facet_item`.
+
+code
+ The field name identifying the facet field, eg. "tags"
+
+title
+ A callable used to render the title for the facet. The callable must accept
+ one string argument, the `code` argument passed in above. Obviously, this
+ is a bit redundant.
+
+label
+ Renders the human-readable label for each facet value.
+ If defined, this should be a callable that accepts a facet_item's name as a
+ string.
+
+if_empty
+ A string, which if defined, and the list of possible facet items is empty,
+ is displayed in lieu of an empty list.
+
+count_label
+ A callable which accepts an integer, and returns a string. This controls
+ how a facet-item's count is displayed.
+
+-->
<py:def function="facet_sidebar(code, limit=5, label=lambda n: n, title=h.facet_title, if_empty=None, count_label=lambda c: ' (%d)'%c)">
- <div py:if="if_empty is not None or len(h.facet_items(code, limit=limit))" class="facet-box">
+ <?python
+ import logging
+ logging.getLogger('ckan.templates.facets').warning("Deprecated function: ckan/templates/facets.html:facet_sidebar()")
+ ?>
+ <div py:if="if_empty is not None or h.unselected_facet_items(code, limit)" class="facet-box">
<h2>${title(code)}</h2>
<ul class="facet-options">
- <li py:for="name, count in h.facet_items(code, limit=limit)"
- py:if="not (code, name) in c.fields">
- <a href="${c.drill_down_url(**{code: name})}">
- <span py:if="'format' in code.lower()">${h.icon(h.format_icon(name))}</span>
- ${label(name)}</a>${count_label(count)}
+ <li py:for="facet_item in h.unselected_facet_items(code, limit)"
+ py:if="not (code, facet_item.name) in c.fields">
+ <a href="${c.drill_down_url(**{code: facet_item.name})}">
+ <span py:if="'format' in code.lower()">${h.icon(h.format_icon(facet_item.name))}</span>
+ ${label(facet_item.name)}</a>${count_label(facet_item['count'])}
</li>
</ul>
- <p py:if="not len(h.facet_items(code, limit=limit))">${if_empty}</p>
+ <p py:if="not h.unselected_facet_items(code, limit)">${if_empty}</p>
</div>
</py:def>
+<!--
+DEPRECATED. Provided only for backward compatibility with existing plugins.
+ Use `facet_li` instead.
+
+Construct a possibly empty list of <li> elements containing links to further
+search results
+
+This is different from `facet_sidebar` and `facet_div` (above) in that it
+requires the caller to wrap up the resulting <li> elements in whatever dom
+element they need. But it *does* allow for filters to displayed in a
+hierarchy.
+
+label
+ Renders the human-readable label for each facet value.
+ If defined, this should be a callable that accepts a facet_item's name as a
+ string.
+
+if_empty
+ If if_empty is not None and there are no facets to filter on, then a single
+ <li> element is generated, with the text specified by if_empty
+
+-->
<py:def function="facet_list_items(code, limit=5, label=lambda n: n, if_empty=None)">
- <!-- Creates a possibly empty list of <li> elements containing links to further search results
-
- This is different from the above function in that it requires the caller to wrap
- up the resulting <li> elements in whatever dom element they need. But it does allow for
- filters to displayed in a hierarchy.
-
- If if_empty is not None and there are no facets to filter on, then a single <li> element
- is generated, with the text specified by if_empty
- -->
- <li py:if="if_empty and len(h.facet_items(code, limit=limit)) == 0">${if_empty}</li>
- <li py:for="name, count in h.facet_items(code, limit=limit)"
- py:if="not (code, name) in c.fields">
- <a href="${c.drill_down_url(**{code: name})}">${label(name)}</a> (${count})
- </li>
+
+ <?python
+ import logging
+ logging.getLogger('ckan.templates.facets').warning("Deprecated function:ckan/templates/facets.html:facet_list_items()")
+ ?>
+ <li py:if="if_empty and not h.unselected_facet_items(code, limit)">${if_empty}</li>
+ <li py:for="facet_item in h.unselected_facet_items(code, limit)">
+ <a href="${c.drill_down_url(**{code: facet_item.name})}">${label(facet_item.name)}</a> (facet_item['count'])
+ </li>
</py:def>
<py:def function="field_list()">
<div class="filter-list" py:if="c.fields">
<div class="filter-entry" py:for="(field, value) in c.fields">
<span class="name">${h.facet_title(field)}</span>
- <span class="value">${value}</span>
+ <span class="value" py:choose="">
+ <py:when test="c.translated_fields and c.translated_fields.has_key((field,value))">
+ ${c.translated_fields[(field,value)]}
+ </py:when>
+ <py:otherwise>
+ ${value}
+ </py:otherwise>
+ </span>
<a href="${c.remove_field(field, value)}">
${h.icon('unfilter')}
</a>
diff --git a/ckan/templates/group/read.html b/ckan/templates/group/read.html
index f3c6b9f..7886bdd 100644
--- a/ckan/templates/group/read.html
+++ b/ckan/templates/group/read.html
@@ -4,8 +4,9 @@
py:strip="">
<xi:include href="../facets.html" />
- <py:def function="page_title">${c.group.display_name}</py:def>
- <py:def function="page_heading">${c.group.display_name}</py:def>
+
+ <py:def function="page_title">${c.group_dict.display_name}</py:def>
+ <py:def function="page_heading">${c.group_dict.display_name}</py:def>
<py:if test="c.group.image_url">
<py:def function="page_logo">${c.group.image_url}</py:def>
</py:if>
@@ -24,8 +25,8 @@
</py:if>
</ul>
</li>
- ${facet_sidebar('tags')}
- ${facet_sidebar('res_format')}
+ ${facet_div('tags', 'Tags')}
+ ${facet_div('res_format', 'Resource Formats')}
</py:match>
<py:match path="content">
diff --git a/ckan/templates/package/search.html b/ckan/templates/package/search.html
index a73bfa2..e503295 100644
--- a/ckan/templates/package/search.html
+++ b/ckan/templates/package/search.html
@@ -20,10 +20,10 @@
<a href="${h.url_for(controller='package', action='new', id=None)}">Register it now</a>.
</p>
</li>
-
- ${facet_sidebar('tags')}
- ${facet_sidebar('res_format')}
- ${facet_sidebar('groups', label=h.group_name_to_title)}
+
+ ${facet_div('tags', 'Tags')}
+ ${facet_div('res_format', 'Resource Formats')}
+ ${facet_div('groups', 'Groups')}
<li class="widget-container widget_text">
<h4>Other access</h4>
diff --git a/ckan/templates/tag/index.html b/ckan/templates/tag/index.html
index 0452d34..17292e9 100644
--- a/ckan/templates/tag/index.html
+++ b/ckan/templates/tag/index.html
@@ -27,7 +27,7 @@
</p>
${c.page.pager(q=c.q)}
- ${tag_list_from_name(c.page.items)}
+ ${tag_list(c.page.items)}
${c.page.pager(q=c.q)}
<p py:if="c.q">
diff --git a/ckan/templates/tag/read.html b/ckan/templates/tag/read.html
index 24997ac..65256f0 100644
--- a/ckan/templates/tag/read.html
+++ b/ckan/templates/tag/read.html
@@ -3,11 +3,11 @@
xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">
- <py:def function="page_title">${c.tag['name']} - Tags</py:def>
- <py:def function="page_heading">Tag: ${c.tag['name']}</py:def>
+ <py:def function="page_title">${c.tag['display_name']} - Tags</py:def>
+ <py:def function="page_heading">Tag: ${c.tag['display_name']}</py:def>
<div py:match="content">
- <p i18n:msg="count, tagname">There are ${len(c.tag['packages'])} datasets tagged with <strong>${c.tag['name']}</strong>:</p>
+ <p i18n:msg="count, tagname">There are ${len(c.tag['packages'])} datasets tagged with <strong>${c.tag['display_name']}</strong>:</p>
${package_list_from_dict(c.tag['packages'])}
</div>
diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py
index c6ceb0b..09342d2 100644
--- a/ckan/tests/lib/test_dictization.py
+++ b/ckan/tests/lib/test_dictization.py
@@ -102,9 +102,13 @@ def setup_class(cls):
u'webstore_last_updated': None,
u'webstore_url': None}],
'state': u'active',
- 'tags': [{'name': u'Flexible \u30a1', 'state': u'active'},
- {'name': u'russian', 'state': u'active'},
- {'name': u'tolstoy', 'state': u'active'}],
+ 'tags': [{'name': u'Flexible \u30a1',
+ 'display_name': u'Flexible \u30a1',
+ 'state': u'active'},
+ {'name': u'russian', 'display_name': u'russian',
+ 'state': u'active'},
+ {'name': u'tolstoy', 'display_name': u'tolstoy',
+ 'state': u'active'}],
'title': u'A Novel By Tolstoy',
'url': u'http://www.annakarenina.com',
'version': u'0.7a'}
@@ -678,6 +682,7 @@ def test_13_get_package_in_past(self):
second_dictized['name'] = u'annakarenina_changed2'
second_dictized['resources'][0]['url'] = u'new_url2'
second_dictized['tags'][0]['name'] = u'new_tag'
+ second_dictized['tags'][0]['display_name'] = u'new_tag'
second_dictized['extras'][0]['value'] = u'"new_value"'
second_dictized['state'] = 'pending'
@@ -705,7 +710,7 @@ def test_13_get_package_in_past(self):
u'webstore_last_updated': None,
u'webstore_url': None})
- third_dictized['tags'].insert(1, {'name': u'newnew_tag', 'state': 'active'})
+ third_dictized['tags'].insert(1, {'name': u'newnew_tag', 'display_name': u'newnew_tag', 'state': 'active'})
third_dictized['extras'].insert(0, {'key': 'david',
'value': u'"new_value"',
'state': u'active'})
@@ -927,8 +932,9 @@ def test_16_group_dictized(self):
result = self.remove_changable_columns(group_dictized)
result['packages'] = sorted(result['packages'], key=lambda x: x['name'])
- assert result == expected, pformat(result)
-
+ assert_equal(sorted(result.keys()), sorted(expected.keys()))
+ for key in result:
+ assert_equal(sorted(result[key]), sorted(expected[key]))
def test_17_group_apis_to_dict(self):
diff --git a/ckan/tests/logic/__init__.py b/ckan/tests/logic/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py
index 8949ab9..a3294f1 100644
--- a/ckan/tests/logic/test_action.py
+++ b/ckan/tests/logic/test_action.py
@@ -1542,6 +1542,27 @@ def test_38_user_role_bulk_update(self):
{'domain_object': anna.id})
assert_equal(results['roles'], roles_after['roles'])
+ def test_40_task_resource_status(self):
+
+ import ckan.lib.celery_app as celery_app
+ backend = celery_app.celery.backend
+ ##This creates the database tables as a side effect, can not see another way
+ ##to make tables unless you actually create a task.
+ celery_result_session = backend.ResultSession()
+
+ ## need to do inserts as setting up an embedded celery is too much for these tests
+ model.Session.connection().execute(
+ '''INSERT INTO task_status (id, entity_id, entity_type, task_type, key, value, state, error, last_updated) VALUES ('5753adae-cd0d-4327-915d-edd832d1c9a3', '749cdcf2-3fc8-44ae-aed0-5eff8cc5032c', 'resource', 'qa', 'celery_task_id', '51f2105d-85b1-4393-b821-ac11475919d9', NULL, '', '2012-04-20 21:32:45.553986');
+ INSERT INTO celery_taskmeta (id, task_id, status, result, date_done, traceback) VALUES (2, '51f2105d-85b1-4393-b821-ac11475919d9', 'FAILURE', '52e', '2012-04-20 21:33:01.622557', 'Traceback')'''
+ )
+ model.Session.commit()
+ res = self.app.post('/api/action/resource_status_show',
+ params=json.dumps({'id': '749cdcf2-3fc8-44ae-aed0-5eff8cc5032c'}),
+ status=200)
+
+ assert json.loads(res.body) == {"help": None, "success": True, "result": [{"status": "FAILURE", "entity_id": "749cdcf2-3fc8-44ae-aed0-5eff8cc5032c", "task_type": "qa", "last_updated": "2012-04-20T21:32:45.553986", "date_done": "2012-04-20T21:33:01.622557", "entity_type": "resource", "traceback": "Traceback", "value": "51f2105d-85b1-4393-b821-ac11475919d9", "state": None, "key": "celery_task_id", "error": "", "id": "5753adae-cd0d-4327-915d-edd832d1c9a3"}]}
+
+
class TestActionTermTranslation(WsgiAppCase):
@classmethod
@@ -1594,7 +1615,7 @@ def test_1_update_single(self):
assert json.loads(res.body)['success']
- postparams = '%s=1' % json.dumps({"term" : "moo"})
+ postparams = '%s=1' % json.dumps({"terms" : ["moo"]})
res = self.app.post('/api/action/term_translation_show', params=postparams,
extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
@@ -1628,7 +1649,7 @@ def test_2_update_many(self):
assert json.loads(res.body)['result']['success'] == '3 rows updated', json.loads(res.body)
- postparams = '%s=1' % json.dumps({"term" : "many"})
+ postparams = '%s=1' % json.dumps({"terms" : ["many"]})
res = self.app.post('/api/action/term_translation_show', params=postparams,
extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
status=200)
diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py
new file mode 100644
index 0000000..a4696bd
--- /dev/null
+++ b/ckanext/multilingual/plugin.py
@@ -0,0 +1,276 @@
+import sets
+import ckan
+from ckan.plugins import SingletonPlugin, implements, IPackageController
+from ckan.plugins import IGroupController, ITagController
+import pylons
+import ckan.logic.action.get as action_get
+from pylons import config
+
+LANGS = ['en', 'fr', 'de', 'es', 'it', 'nl', 'ro', 'pt', 'pl']
+
+def translate_data_dict(data_dict):
+ '''Return the given dict (e.g. a dataset dict) with as many of its fields
+ as possible translated into the desired or the fallback language.
+
+ '''
+ desired_lang_code = pylons.request.environ['CKAN_LANG']
+ fallback_lang_code = pylons.config.get('ckan.locale_default', 'en')
+
+ # Get a flattened copy of data_dict to do the translation on.
+ flattened = ckan.lib.navl.dictization_functions.flatten_dict(
+ data_dict)
+
+ # Get a simple flat list of all the terms to be translated, from the
+ # flattened data dict.
+ terms = sets.Set()
+ for (key, value) in flattened.items():
+ if value in (None, True, False):
+ continue
+ elif isinstance(value, basestring):
+ terms.add(value)
+ elif isinstance(value, int):
+ continue
+ else:
+ for item in value:
+ terms.add(item)
+
+ # Get the translations of all the terms (as a list of dictionaries).
+ translations = ckan.logic.action.get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': terms,
+ 'lang_codes': (desired_lang_code, fallback_lang_code)})
+
+ # Transform the translations into a more convenient structure.
+ desired_translations = {}
+ fallback_translations = {}
+ for translation in translations:
+ if translation['lang_code'] == desired_lang_code:
+ desired_translations[translation['term']] = (
+ translation['term_translation'])
+ else:
+ assert translation['lang_code'] == fallback_lang_code
+ fallback_translations[translation['term']] = (
+ translation['term_translation'])
+
+ # Make a copy of the flattened data dict with all the terms replaced by
+ # their translations, where available.
+ translated_flattened = {}
+ for (key, value) in flattened.items():
+
+ # Don't translate names that are used for form URLs.
+ if key == ('name',):
+ translated_flattened[key] = value
+ elif (key[0] in ('tags', 'groups') and len(key) == 3
+ and key[2] == 'name'):
+ translated_flattened[key] = value
+
+ elif value in (None, True, False):
+ # Don't try to translate values that aren't strings.
+ translated_flattened[key] = value
+
+ elif isinstance(value, basestring):
+ if desired_translations.has_key(value):
+ translated_flattened[key] = desired_translations[value]
+ else:
+ translated_flattened[key] = fallback_translations.get(
+ value, value)
+
+ elif isinstance(value, int):
+ translated_flattened[key] = value
+
+ else:
+ translated_value = []
+ for item in value:
+ if desired_translations.has_key(value):
+ translated_flattened[key] = desired_translations[value]
+ else:
+ translated_flattened[key] = fallback_translations.get(
+ value, value)
+ translated_flattened[key] = translated_value
+
+ # Finally unflatten and return the translated data dict.
+ translated_data_dict = (ckan.lib.navl.dictization_functions
+ .unflatten(translated_flattened))
+ return translated_data_dict
+
+KEYS_TO_IGNORE = ['state', 'revision_id', 'id', #title done seperately
+ 'metadata_created', 'metadata_modified', 'site_id']
+
+class MultilingualDataset(SingletonPlugin):
+ implements(IPackageController, inherit=True)
+
+ def before_index(self, search_data):
+
+ default_lang = search_data.get(
+ 'lang_code',
+ pylons.config.get('ckan.locale_default', 'en')
+ )
+
+ ## translate title
+ title = search_data.get('title')
+ search_data['title_' + default_lang] = title
+ title_translations = action_get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': [title],
+ 'lang_codes': LANGS})
+
+ for translation in title_translations:
+ title_field = 'title_' + translation['lang_code']
+ search_data[title_field] = translation['term_translation']
+
+ ## translate rest
+ all_terms = []
+ for key, value in search_data.iteritems():
+ if key in KEYS_TO_IGNORE or key.startswith('title'):
+ continue
+ if isinstance(value, list):
+ all_terms.extend(value)
+ elif value in (None, True, False):
+ continue
+ else:
+ all_terms.append(value)
+
+ field_translations = action_get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': all_terms,
+ 'lang_codes': LANGS})
+
+ text_field_items = dict(('text_' + lang, []) for lang in LANGS)
+
+ text_field_items['text_' + default_lang].extend(all_terms)
+
+ for translation in sorted(field_translations):
+ lang_field = 'text_' + translation['lang_code']
+ text_field_items[lang_field].append(translation['term_translation'])
+
+ for key, value in text_field_items.iteritems():
+ search_data[key] = ' '.join(value)
+
+ return search_data
+
+ def before_search(self, search_params):
+ lang_set = set(LANGS)
+ current_lang = pylons.request.environ['CKAN_LANG']
+ # fallback to default locale if locale not in suported langs
+ if not current_lang in lang_set:
+ current_lang = config.get('ckan.locale_default')
+ # fallback to english if default locale is not supported
+ if not current_lang in lang_set:
+ current_lang = 'en'
+ # treat current lang differenly so remove from set
+ lang_set.remove(current_lang)
+
+ # weight current lang more highly
+ query_fields = 'title_%s^8 text_%s^4' % (current_lang, current_lang)
+
+ for lang in lang_set:
+ query_fields += ' title_%s^2 text_%s' % (lang, lang)
+
+ search_params['qf'] = query_fields
+
+ return search_params
+
+ def after_search(self, search_results, search_params):
+
+ # Translate the unselected search facets.
+ facets = search_results.get('search_facets')
+ if not facets:
+ return search_results
+
+ desired_lang_code = pylons.request.environ['CKAN_LANG']
+ fallback_lang_code = pylons.config.get('ckan.locale_default', 'en')
+
+ # Look up translations for all of the facets in one db query.
+ terms = sets.Set()
+ for facet in facets.values():
+ for item in facet['items']:
+ terms.add(item['display_name'])
+ translations = ckan.logic.action.get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': terms,
+ 'lang_codes': (desired_lang_code, fallback_lang_code)})
+
+ # Replace facet display names with translated ones.
+ for facet in facets.values():
+ for item in facet['items']:
+ matching_translations = [translation for
+ translation in translations
+ if translation['term'] == item['display_name']
+ and translation['lang_code'] == desired_lang_code]
+ if not matching_translations:
+ matching_translations = [translation for
+ translation in translations
+ if translation['term'] == item['display_name']
+ and translation['lang_code'] == fallback_lang_code]
+ if matching_translations:
+ assert len(matching_translations) == 1
+ item['display_name'] = (
+ matching_translations[0]['term_translation'])
+
+ return search_results
+
+ def before_view(self, dataset_dict):
+
+ # Translate any selected search facets (e.g. if we are rendering a
+ # group read page or the dataset index page): lookup translations of
+ # all the terms in c.fields (c.fields contains the selected facets)
+ # and save them in c.translated_fields where the templates can
+ # retrieve them later.
+ c = pylons.c
+ desired_lang_code = pylons.request.environ['CKAN_LANG']
+ fallback_lang_code = pylons.config.get('ckan.locale_default', 'en')
+ terms = [value for param, value in c.fields]
+ translations = ckan.logic.action.get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': terms,
+ 'lang_codes': (desired_lang_code, fallback_lang_code)})
+ c.translated_fields = {}
+ for param, value in c.fields:
+ matching_translations = [translation for translation in
+ translations if translation['term'] == value and
+ translation['lang_code'] == desired_lang_code]
+ if not matching_translations:
+ matching_translations = [translation for translation in
+ translations if translation['term'] == value and
+ translation['lang_code'] == fallback_lang_code]
+ if matching_translations:
+ assert len(matching_translations) == 1
+ translation = matching_translations[0]['term_translation']
+ c.translated_fields[(param, value)] = translation
+
+ # Now translate the fields of the dataset itself.
+ return translate_data_dict(dataset_dict)
+
+class MultilingualGroup(SingletonPlugin):
+ '''The MultilingualGroup plugin translates group names and other group
+ fields on group read pages and on the group index page.
+
+ For example on the page /de/group/david the title "Dave's Books" at the
+ top of the page might be translated to "Dave's Bucher".
+
+ Datasets are also shown on group pages, but these are translated by the
+ MultilingualDataset plugin.
+
+ '''
+ implements(IGroupController, inherit=True)
+
+ def before_view(self, data_dict):
+ translated_data_dict = translate_data_dict(data_dict)
+ return translated_data_dict
+
+class MultilingualTag(SingletonPlugin):
+ '''The MultilingualTag plugin translates tag names on tag read pages and
+ on the tag index page.
+
+ For example on the page /de/tag/tolstoy the title "Tag: tolstoy" at the
+ top of the page might be translated to "Tag: Tolstoi".
+
+ Datasets are also shown on tag pages, but these are translated by the
+ MultilingualDataset plugin.
+
+ '''
+ implements(ITagController, inherit=True)
+
+ def before_view(self, data_dict):
+ translated_data_dict = translate_data_dict(data_dict)
+ return translated_data_dict
diff --git a/ckanext/multilingual/solr/dutch_stop.txt b/ckanext/multilingual/solr/dutch_stop.txt
new file mode 100644
index 0000000..f4d61f5
--- /dev/null
+++ b/ckanext/multilingual/solr/dutch_stop.txt
@@ -0,0 +1,117 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A Dutch stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | This is a ranked list (commonest to rarest) of stopwords derived from
+ | a large sample of Dutch text.
+
+ | Dutch stop words frequently exhibit homonym clashes. These are indicated
+ | clearly below.
+
+de | the
+en | and
+van | of, from
+ik | I, the ego
+te | (1) chez, at etc, (2) to, (3) too
+dat | that, which
+die | that, those, who, which
+in | in, inside
+een | a, an, one
+hij | he
+het | the, it
+niet | not, nothing, naught
+zijn | (1) to be, being, (2) his, one's, its
+is | is
+was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river
+op | on, upon, at, in, up, used up
+aan | on, upon, to (as dative)
+met | with, by
+als | like, such as, when
+voor | (1) before, in front of, (2) furrow
+had | had, past tense all persons sing. of 'hebben' (have)
+er | there
+maar | but, only
+om | round, about, for etc
+hem | him
+dan | then
+zou | should/would, past tense all persons sing. of 'zullen'
+of | or, whether, if
+wat | what, something, anything
+mijn | possessive and noun 'mine'
+men | people, 'one'
+dit | this
+zo | so, thus, in this way
+door | through by
+over | over, across
+ze | she, her, they, them
+zich | oneself
+bij | (1) a bee, (2) by, near, at
+ook | also, too
+tot | till, until
+je | you
+mij | me
+uit | out of, from
+der | Old Dutch form of 'van der' still found in surnames
+daar | (1) there, (2) because
+haar | (1) her, their, them, (2) hair
+naar | (1) unpleasant, unwell etc, (2) towards, (3) as
+heb | present first person sing. of 'to have'
+hoe | how, why
+heeft | present third person sing. of 'to have'
+hebben | 'to have' and various parts thereof
+deze | this
+u | you
+want | (1) for, (2) mitten, (3) rigging
+nog | yet, still
+zal | 'shall', first and third person sing. of verb 'zullen' (will)
+me | me
+zij | she, they
+nu | now
+ge | 'thou', still used in Belgium and south Netherlands
+geen | none
+omdat | because
+iets | something, somewhat
+worden | to become, grow, get
+toch | yet, still
+al | all, every, each
+waren | (1) 'were' (2) to wander, (3) wares, (3)
+veel | much, many
+meer | (1) more, (2) lake
+doen | to do, to make
+toen | then, when
+moet | noun 'spot/mote' and present form of 'to must'
+ben | (1) am, (2) 'are' in interrogative second person singular of 'to be'
+zonder | without
+kan | noun 'can' and present form of 'to be able'
+hun | their, them
+dus | so, consequently
+alles | all, everything, anything
+onder | under, beneath
+ja | yes, of course
+eens | once, one day
+hier | here
+wie | who
+werd | imperfect third person sing. of 'become'
+altijd | always
+doch | yet, but etc
+wordt | present third person sing. of 'become'
+wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans
+kunnen | to be able
+ons | us/our
+zelf | self
+tegen | against, towards, at
+na | after, near
+reeds | already
+wil | (1) present tense of 'want', (2) 'will', noun, (3) fender
+kon | could; past tense of 'to be able'
+niets | nothing
+uw | your
+iemand | somebody
+geweest | been; past participle of 'be'
+andere | other
diff --git a/ckanext/multilingual/solr/english_stop.txt b/ckanext/multilingual/solr/english_stop.txt
new file mode 100644
index 0000000..821c03e
--- /dev/null
+++ b/ckanext/multilingual/solr/english_stop.txt
@@ -0,0 +1,317 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/english/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | An English stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | Many of the forms below are quite rare (e.g. "yourselves") but included for
+ | completeness.
+
+ | PRONOUNS FORMS
+ | 1st person sing
+
+i | subject, always in upper case of course
+
+me | object
+my | possessive adjective
+ | the possessive pronoun `mine' is best suppressed, because of the
+ | sense of coal-mine etc.
+myself | reflexive
+ | 1st person plural
+we | subject
+
+| us | object
+ | care is required here because US = United States. It is usually
+ | safe to remove it if it is in lower case.
+our | possessive adjective
+ours | possessive pronoun
+ourselves | reflexive
+ | second person (archaic `thou' forms not included)
+you | subject and object
+your | possessive adjective
+yours | possessive pronoun
+yourself | reflexive (singular)
+yourselves | reflexive (plural)
+ | third person singular
+he | subject
+him | object
+his | possessive adjective and pronoun
+himself | reflexive
+
+she | subject
+her | object and possessive adjective
+hers | possessive pronoun
+herself | reflexive
+
+it | subject and object
+its | possessive adjective
+itself | reflexive
+ | third person plural
+they | subject
+them | object
+their | possessive adjective
+theirs | possessive pronoun
+themselves | reflexive
+ | other forms (demonstratives, interrogatives)
+what
+which
+who
+whom
+this
+that
+these
+those
+
+ | VERB FORMS (using F.R. Palmer's nomenclature)
+ | BE
+am | 1st person, present
+is | -s form (3rd person, present)
+are | present
+was | 1st person, past
+were | past
+be | infinitive
+been | past participle
+being | -ing form
+ | HAVE
+have | simple
+has | -s form
+had | past
+having | -ing form
+ | DO
+do | simple
+does | -s form
+did | past
+doing | -ing form
+
+ | The forms below are, I believe, best omitted, because of the significant
+ | homonym forms:
+
+ | He made a WILL
+ | old tin CAN
+ | merry month of MAY
+ | a smell of MUST
+ | fight the good fight with all thy MIGHT
+
+ | would, could, should, ought might however be included
+
+ | | AUXILIARIES
+ | | WILL
+ |will
+
+would
+
+ | | SHALL
+ |shall
+
+should
+
+ | | CAN
+ |can
+
+could
+
+ | | MAY
+ |may
+ |might
+ | | MUST
+ |must
+ | | OUGHT
+
+ought
+
+ | COMPOUND FORMS, increasingly encountered nowadays in 'formal' writing
+ | pronoun + verb
+
+i'm
+you're
+he's
+she's
+it's
+we're
+they're
+i've
+you've
+we've
+they've
+i'd
+you'd
+he'd
+she'd
+we'd
+they'd
+i'll
+you'll
+he'll
+she'll
+we'll
+they'll
+
+ | verb + negation
+
+isn't
+aren't
+wasn't
+weren't
+hasn't
+haven't
+hadn't
+doesn't
+don't
+didn't
+
+ | auxiliary + negation
+
+won't
+wouldn't
+shan't
+shouldn't
+can't
+cannot
+couldn't
+mustn't
+
+ | miscellaneous forms
+
+let's
+that's
+who's
+what's
+here's
+there's
+when's
+where's
+why's
+how's
+
+ | rarer forms
+
+ | daren't needn't
+
+ | doubtful forms
+
+ | oughtn't mightn't
+
+ | ARTICLES
+a
+an
+the
+
+ | THE REST (Overlap among prepositions, conjunctions, adverbs etc is so
+ | high, that classification is pointless.)
+and
+but
+if
+or
+because
+as
+until
+while
+
+of
+at
+by
+for
+with
+about
+against
+between
+into
+through
+during
+before
+after
+above
+below
+to
+from
+up
+down
+in
+out
+on
+off
+over
+under
+
+again
+further
+then
+once
+
+here
+there
+when
+where
+why
+how
+
+all
+any
+both
+each
+few
+more
+most
+other
+some
+such
+
+no
+nor
+not
+only
+own
+same
+so
+than
+too
+very
+
+ | Just for the record, the following words are among the commonest in English
+
+ | one
+ | every
+ | least
+ | less
+ | many
+ | now
+ | ever
+ | never
+ | say
+ | says
+ | said
+ | also
+ | get
+ | go
+ | goes
+ | just
+ | made
+ | make
+ | put
+ | see
+ | seen
+ | whether
+ | like
+ | well
+ | back
+ | even
+ | still
+ | way
+ | take
+ | since
+ | another
+ | however
+ | two
+ | three
+ | four
+ | five
+ | first
+ | second
+ | new
+ | old
+ | high
+ | long
diff --git a/ckanext/multilingual/solr/fr_elision.txt b/ckanext/multilingual/solr/fr_elision.txt
new file mode 100644
index 0000000..4e855c1
--- /dev/null
+++ b/ckanext/multilingual/solr/fr_elision.txt
@@ -0,0 +1,8 @@
+c
+d
+j
+l
+m
+n
+s
+t
diff --git a/ckanext/multilingual/solr/french_stop.txt b/ckanext/multilingual/solr/french_stop.txt
new file mode 100644
index 0000000..c00837e
--- /dev/null
+++ b/ckanext/multilingual/solr/french_stop.txt
@@ -0,0 +1,183 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A French stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+au | a + le
+aux | a + les
+avec | with
+ce | this
+ces | these
+dans | with
+de | of
+des | de + les
+du | de + le
+elle | she
+en | `of them' etc
+et | and
+eux | them
+il | he
+je | I
+la | the
+le | the
+leur | their
+lui | him
+ma | my (fem)
+mais | but
+me | me
+même | same; as in moi-même (myself) etc
+mes | me (pl)
+moi | me
+mon | my (masc)
+ne | not
+nos | our (pl)
+notre | our
+nous | we
+on | one
+ou | where
+par | by
+pas | not
+pour | for
+qu | que before vowel
+que | that
+qui | who
+sa | his, her (fem)
+se | oneself
+ses | his (pl)
+son | his, her (masc)
+sur | on
+ta | thy (fem)
+te | thee
+tes | thy (pl)
+toi | thee
+ton | thy (masc)
+tu | thou
+un | a
+une | a
+vos | your (pl)
+votre | your
+vous | you
+
+ | single letter forms
+
+c | c'
+d | d'
+j | j'
+l | l'
+à | to, at
+m | m'
+n | n'
+s | s'
+t | t'
+y | there
+
+ | forms of être (not including the infinitive):
+été
+étée
+étées
+étés
+étant
+suis
+es
+est
+sommes
+êtes
+sont
+serai
+seras
+sera
+serons
+serez
+seront
+serais
+serait
+serions
+seriez
+seraient
+étais
+était
+étions
+étiez
+étaient
+fus
+fut
+fûmes
+fûtes
+furent
+sois
+soit
+soyons
+soyez
+soient
+fusse
+fusses
+fût
+fussions
+fussiez
+fussent
+
+ | forms of avoir (not including the infinitive):
+ayant
+eu
+eue
+eues
+eus
+ai
+as
+avons
+avez
+ont
+aurai
+auras
+aura
+aurons
+aurez
+auront
+aurais
+aurait
+aurions
+auriez
+auraient
+avais
+avait
+avions
+aviez
+avaient
+eut
+eûmes
+eûtes
+eurent
+aie
+aies
+ait
+ayons
+ayez
+aient
+eusse
+eusses
+eût
+eussions
+eussiez
+eussent
+
+ | Later additions (from Jean-Christophe Deschamps)
+ceci | this
+celà | that
+cet | this
+cette | this
+ici | here
+ils | they
+les | the (pl)
+leurs | their (pl)
+quel | which
+quels | which
+quelle | which
+quelles | which
+sans | without
+soi | oneself
+
diff --git a/ckanext/multilingual/solr/german_stop.txt b/ckanext/multilingual/solr/german_stop.txt
new file mode 100644
index 0000000..f770384
--- /dev/null
+++ b/ckanext/multilingual/solr/german_stop.txt
@@ -0,0 +1,292 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A German stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | The number of forms in this list is reduced significantly by passing it
+ | through the German stemmer.
+
+
+aber | but
+
+alle | all
+allem
+allen
+aller
+alles
+
+als | than, as
+also | so
+am | an + dem
+an | at
+
+ander | other
+andere
+anderem
+anderen
+anderer
+anderes
+anderm
+andern
+anderr
+anders
+
+auch | also
+auf | on
+aus | out of
+bei | by
+bin | am
+bis | until
+bist | art
+da | there
+damit | with it
+dann | then
+
+der | the
+den
+des
+dem
+die
+das
+
+daß | that
+
+derselbe | the same
+derselben
+denselben
+desselben
+demselben
+dieselbe
+dieselben
+dasselbe
+
+dazu | to that
+
+dein | thy
+deine
+deinem
+deinen
+deiner
+deines
+
+denn | because
+
+derer | of those
+dessen | of him
+
+dich | thee
+dir | to thee
+du | thou
+
+dies | this
+diese
+diesem
+diesen
+dieser
+dieses
+
+
+doch | (several meanings)
+dort | (over) there
+
+
+durch | through
+
+ein | a
+eine
+einem
+einen
+einer
+eines
+
+einig | some
+einige
+einigem
+einigen
+einiger
+einiges
+
+einmal | once
+
+er | he
+ihn | him
+ihm | to him
+
+es | it
+etwas | something
+
+euer | your
+eure
+eurem
+euren
+eurer
+eures
+
+für | for
+gegen | towards
+gewesen | p.p. of sein
+hab | have
+habe | have
+haben | have
+hat | has
+hatte | had
+hatten | had
+hier | here
+hin | there
+hinter | behind
+
+ich | I
+mich | me
+mir | to me
+
+
+ihr | you, to her
+ihre
+ihrem
+ihren
+ihrer
+ihres
+euch | to you
+
+im | in + dem
+in | in
+indem | while
+ins | in + das
+ist | is
+
+jede | each, every
+jedem
+jeden
+jeder
+jedes
+
+jene | that
+jenem
+jenen
+jener
+jenes
+
+jetzt | now
+kann | can
+
+kein | no
+keine
+keinem
+keinen
+keiner
+keines
+
+können | can
+könnte | could
+machen | do
+man | one
+
+manche | some, many a
+manchem
+manchen
+mancher
+manches
+
+mein | my
+meine
+meinem
+meinen
+meiner
+meines
+
+mit | with
+muss | must
+musste | had to
+nach | to(wards)
+nicht | not
+nichts | nothing
+noch | still, yet
+nun | now
+nur | only
+ob | whether
+oder | or
+ohne | without
+sehr | very
+
+sein | his
+seine
+seinem
+seinen
+seiner
+seines
+
+selbst | self
+sich | herself
+
+sie | they, she
+ihnen | to them
+
+sind | are
+so | so
+
+solche | such
+solchem
+solchen
+solcher
+solches
+
+soll | shall
+sollte | should
+sondern | but
+sonst | else
+über | over
+um | about, around
+und | and
+
+uns | us
+unse
+unsem
+unsen
+unser
+unses
+
+unter | under
+viel | much
+vom | von + dem
+von | from
+vor | before
+während | while
+war | was
+waren | were
+warst | wast
+was | what
+weg | away, off
+weil | because
+weiter | further
+
+welche | which
+welchem
+welchen
+welcher
+welches
+
+wenn | when
+werde | will
+werden | will
+wie | how
+wieder | again
+will | want
+wir | we
+wird | will
+wirst | willst
+wo | where
+wollen | want
+wollte | wanted
+würde | would
+würden | would
+zu | to
+zum | zu + dem
+zur | zu + der
+zwar | indeed
+zwischen | between
+
diff --git a/ckanext/multilingual/solr/greek_stopwords.txt b/ckanext/multilingual/solr/greek_stopwords.txt
new file mode 100644
index 0000000..1a08d31
--- /dev/null
+++ b/ckanext/multilingual/solr/greek_stopwords.txt
@@ -0,0 +1,76 @@
+# Lucene Greek Stopwords list
+ο
+η
+το
+οι
+τα
+του
+τησ
+των
+τον
+την
+και
+κι
+κ
+ειμαι
+εισαι
+ειναι
+ειμαστε
+ειστε
+στο
+στον
+στη
+στην
+μα
+αλλα
+απο
+για
+προσ
+με
+σε
+ωσ
+παρα
+αντι
+κατα
+μετα
+θα
+να
+δε
+δεν
+μη
+μην
+επι
+ενω
+εαν
+αν
+τοτε
+που
+πωσ
+ποιοσ
+ποια
+ποιο
+ποιοι
+ποιεσ
+ποιων
+ποιουσ
+αυτοσ
+αυτη
+αυτο
+αυτοι
+αυτων
+αυτουσ
+αυτεσ
+αυτα
+εκεινοσ
+εκεινη
+εκεινο
+εκεινοι
+εκεινεσ
+εκεινα
+εκεινων
+εκεινουσ
+οπωσ
+ομωσ
+ισωσ
+οσο
+οτι
diff --git a/ckanext/multilingual/solr/italian_stop.txt b/ckanext/multilingual/solr/italian_stop.txt
new file mode 100644
index 0000000..4cb5b08
--- /dev/null
+++ b/ckanext/multilingual/solr/italian_stop.txt
@@ -0,0 +1,301 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | An Italian stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ad | a (to) before vowel
+al | a + il
+allo | a + lo
+ai | a + i
+agli | a + gli
+all | a + l'
+agl | a + gl'
+alla | a + la
+alle | a + le
+con | with
+col | con + il
+coi | con + i (forms collo, cogli etc are now very rare)
+da | from
+dal | da + il
+dallo | da + lo
+dai | da + i
+dagli | da + gli
+dall | da + l'
+dagl | da + gll'
+dalla | da + la
+dalle | da + le
+di | of
+del | di + il
+dello | di + lo
+dei | di + i
+degli | di + gli
+dell | di + l'
+degl | di + gl'
+della | di + la
+delle | di + le
+in | in
+nel | in + el
+nello | in + lo
+nei | in + i
+negli | in + gli
+nell | in + l'
+negl | in + gl'
+nella | in + la
+nelle | in + le
+su | on
+sul | su + il
+sullo | su + lo
+sui | su + i
+sugli | su + gli
+sull | su + l'
+sugl | su + gl'
+sulla | su + la
+sulle | su + le
+per | through, by
+tra | among
+contro | against
+io | I
+tu | thou
+lui | he
+lei | she
+noi | we
+voi | you
+loro | they
+mio | my
+mia |
+miei |
+mie |
+tuo |
+tua |
+tuoi | thy
+tue |
+suo |
+sua |
+suoi | his, her
+sue |
+nostro | our
+nostra |
+nostri |
+nostre |
+vostro | your
+vostra |
+vostri |
+vostre |
+mi | me
+ti | thee
+ci | us, there
+vi | you, there
+lo | him, the
+la | her, the
+li | them
+le | them, the
+gli | to him, the
+ne | from there etc
+il | the
+un | a
+uno | a
+una | a
+ma | but
+ed | and
+se | if
+perché | why, because
+anche | also
+come | how
+dov | where (as dov')
+dove | where
+che | who, that
+chi | who
+cui | whom
+non | not
+più | more
+quale | who, that
+quanto | how much
+quanti |
+quanta |
+quante |
+quello | that
+quelli |
+quella |
+quelle |
+questo | this
+questi |
+questa |
+queste |
+si | yes
+tutto | all
+tutti | all
+
+ | single letter forms:
+
+a | at
+c | as c' for ce or ci
+e | and
+i | the
+l | as l'
+o | or
+
+ | forms of avere, to have (not including the infinitive):
+
+ho
+hai
+ha
+abbiamo
+avete
+hanno
+abbia
+abbiate
+abbiano
+avrò
+avrai
+avrà
+avremo
+avrete
+avranno
+avrei
+avresti
+avrebbe
+avremmo
+avreste
+avrebbero
+avevo
+avevi
+aveva
+avevamo
+avevate
+avevano
+ebbi
+avesti
+ebbe
+avemmo
+aveste
+ebbero
+avessi
+avesse
+avessimo
+avessero
+avendo
+avuto
+avuta
+avuti
+avute
+
+ | forms of essere, to be (not including the infinitive):
+sono
+sei
+è
+siamo
+siete
+sia
+siate
+siano
+sarò
+sarai
+sarà
+saremo
+sarete
+saranno
+sarei
+saresti
+sarebbe
+saremmo
+sareste
+sarebbero
+ero
+eri
+era
+eravamo
+eravate
+erano
+fui
+fosti
+fu
+fummo
+foste
+furono
+fossi
+fosse
+fossimo
+fossero
+essendo
+
+ | forms of fare, to do (not including the infinitive, fa, fat-):
+faccio
+fai
+facciamo
+fanno
+faccia
+facciate
+facciano
+farò
+farai
+farà
+faremo
+farete
+faranno
+farei
+faresti
+farebbe
+faremmo
+fareste
+farebbero
+facevo
+facevi
+faceva
+facevamo
+facevate
+facevano
+feci
+facesti
+fece
+facemmo
+faceste
+fecero
+facessi
+facesse
+facessimo
+facessero
+facendo
+
+ | forms of stare, to be (not including the infinitive):
+sto
+stai
+sta
+stiamo
+stanno
+stia
+stiate
+stiano
+starò
+starai
+starà
+staremo
+starete
+staranno
+starei
+staresti
+starebbe
+staremmo
+stareste
+starebbero
+stavo
+stavi
+stava
+stavamo
+stavate
+stavano
+stetti
+stesti
+stette
+stemmo
+steste
+stettero
+stessi
+stesse
+stessimo
+stessero
+stando
diff --git a/ckanext/multilingual/solr/polish_stop.txt b/ckanext/multilingual/solr/polish_stop.txt
new file mode 100644
index 0000000..167e9e0
--- /dev/null
+++ b/ckanext/multilingual/solr/polish_stop.txt
@@ -0,0 +1,186 @@
+# This file was created from the carrot2 project and is distributed under the BSD license.
+# See http://project.carrot2.org/license.html
+# Also see http://www.opensource.org/licenses/bsd-license.html
+# From trunk/core/carrot2-util-text/src-resources/stopwords.pl
+vol
+o.o.
+mgr
+godz
+zł
+www
+pl
+ul
+tel
+hab
+prof
+inż
+dr
+i
+u
+aby
+albo
+ale
+ani
+aż
+bardzo
+bez
+bo
+bowiem
+by
+byli
+bym
+był
+była
+było
+były
+być
+będzie
+będą
+chce
+choć
+co
+coraz
+coś
+czy
+czyli
+często
+dla
+do
+gdy
+gdyby
+gdyż
+gdzie
+go
+ich
+im
+inne
+iż
+ja
+jak
+jakie
+jako
+je
+jednak
+jednym
+jedynie
+jego
+jej
+jest
+jeszcze
+jeśli
+jeżeli
+już
+ją
+kiedy
+kilku
+kto
+która
+które
+którego
+której
+który
+których
+którym
+którzy
+lat
+lecz
+lub
+ma
+mają
+mamy
+mi
+miał
+mimo
+mnie
+mogą
+może
+można
+mu
+musi
+na
+nad
+nam
+nas
+nawet
+nic
+nich
+nie
+niej
+nim
+niż
+no
+nowe
+np
+nr
+o
+od
+ok
+on
+one
+oraz
+pan
+po
+pod
+ponad
+ponieważ
+poza
+przed
+przede
+przez
+przy
+raz
+razie
+roku
+również
+się
+sobie
+sposób
+swoje
+są
+ta
+tak
+takich
+takie
+także
+tam
+te
+tego
+tej
+temu
+ten
+teraz
+też
+to
+trzeba
+tu
+tych
+tylko
+tym
+tys
+tzw
+tę
+w
+we
+wie
+więc
+wszystko
+wśród
+właśnie
+z
+za
+zaś
+ze
+że
+żeby
+ii
+iii
+iv
+vi
+vii
+viii
+ix
+xi
+xii
+xiii
+xiv
+xv
diff --git a/ckanext/multilingual/solr/portuguese_stop.txt b/ckanext/multilingual/solr/portuguese_stop.txt
new file mode 100644
index 0000000..276c1b4
--- /dev/null
+++ b/ckanext/multilingual/solr/portuguese_stop.txt
@@ -0,0 +1,251 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A Portuguese stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+
+ | The following is a ranked list (commonest to rarest) of stopwords
+ | deriving from a large sample of text.
+
+ | Extra words have been added at the end.
+
+de | of, from
+a | the; to, at; her
+o | the; him
+que | who, that
+e | and
+do | de + o
+da | de + a
+em | in
+um | a
+para | for
+ | é from SER
+com | with
+não | not, no
+uma | a
+os | the; them
+no | em + o
+se | himself etc
+na | em + a
+por | for
+mais | more
+as | the; them
+dos | de + os
+como | as, like
+mas | but
+ | foi from SER
+ao | a + o
+ele | he
+das | de + as
+ | tem from TER
+à | a + a
+seu | his
+sua | her
+ou | or
+ | ser from SER
+quando | when
+muito | much
+ | há from HAV
+nos | em + os; us
+já | already, now
+ | está from EST
+eu | I
+também | also
+só | only, just
+pelo | per + o
+pela | per + a
+até | up to
+isso | that
+ela | he
+entre | between
+ | era from SER
+depois | after
+sem | without
+mesmo | same
+aos | a + os
+ | ter from TER
+seus | his
+quem | whom
+nas | em + as
+me | me
+esse | that
+eles | they
+ | estão from EST
+você | you
+ | tinha from TER
+ | foram from SER
+essa | that
+num | em + um
+nem | nor
+suas | her
+meu | my
+às | a + as
+minha | my
+ | têm from TER
+numa | em + uma
+pelos | per + os
+elas | they
+ | havia from HAV
+ | seja from SER
+qual | which
+ | será from SER
+nós | we
+ | tenho from TER
+lhe | to him, her
+deles | of them
+essas | those
+esses | those
+pelas | per + as
+este | this
+ | fosse from SER
+dele | of him
+
+ | other words. There are many contractions such as naquele = em+aquele,
+ | mo = me+o, but they are rare.
+ | Indefinite article plural forms are also rare.
+
+tu | thou
+te | thee
+vocês | you (plural)
+vos | you
+lhes | to them
+meus | my
+minhas
+teu | thy
+tua
+teus
+tuas
+nosso | our
+nossa
+nossos
+nossas
+
+dela | of her
+delas | of them
+
+esta | this
+estes | these
+estas | these
+aquele | that
+aquela | that
+aqueles | those
+aquelas | those
+isto | this
+aquilo | that
+
+ | forms of estar, to be (not including the infinitive):
+estou
+está
+estamos
+estão
+estive
+esteve
+estivemos
+estiveram
+estava
+estávamos
+estavam
+estivera
+estivéramos
+esteja
+estejamos
+estejam
+estivesse
+estivéssemos
+estivessem
+estiver
+estivermos
+estiverem
+
+ | forms of haver, to have (not including the infinitive):
+hei
+há
+havemos
+hão
+houve
+houvemos
+houveram
+houvera
+houvéramos
+haja
+hajamos
+hajam
+houvesse
+houvéssemos
+houvessem
+houver
+houvermos
+houverem
+houverei
+houverá
+houveremos
+houverão
+houveria
+houveríamos
+houveriam
+
+ | forms of ser, to be (not including the infinitive):
+sou
+somos
+são
+era
+éramos
+eram
+fui
+foi
+fomos
+foram
+fora
+fôramos
+seja
+sejamos
+sejam
+fosse
+fôssemos
+fossem
+for
+formos
+forem
+serei
+será
+seremos
+serão
+seria
+seríamos
+seriam
+
+ | forms of ter, to have (not including the infinitive):
+tenho
+tem
+temos
+tém
+tinha
+tínhamos
+tinham
+tive
+teve
+tivemos
+tiveram
+tivera
+tivéramos
+tenha
+tenhamos
+tenham
+tivesse
+tivéssemos
+tivessem
+tiver
+tivermos
+tiverem
+terei
+terá
+teremos
+terão
+teria
+teríamos
+teriam
diff --git a/ckanext/multilingual/solr/romanian_stop.txt b/ckanext/multilingual/solr/romanian_stop.txt
new file mode 100644
index 0000000..4fdee90
--- /dev/null
+++ b/ckanext/multilingual/solr/romanian_stop.txt
@@ -0,0 +1,233 @@
+# This file was created by Jacques Savoy and is distributed under the BSD license.
+# See http://members.unine.ch/jacques.savoy/clef/index.html.
+# Also see http://www.opensource.org/licenses/bsd-license.html
+acea
+aceasta
+această
+aceea
+acei
+aceia
+acel
+acela
+acele
+acelea
+acest
+acesta
+aceste
+acestea
+aceşti
+aceştia
+acolo
+acum
+ai
+aia
+aibă
+aici
+al
+ăla
+ale
+alea
+ălea
+altceva
+altcineva
+am
+ar
+are
+aş
+aşadar
+asemenea
+asta
+ăsta
+astăzi
+astea
+ăstea
+ăştia
+asupra
+aţi
+au
+avea
+avem
+aveţi
+azi
+bine
+bucur
+bună
+ca
+că
+căci
+când
+care
+cărei
+căror
+cărui
+cât
+câte
+câţi
+către
+câtva
+ce
+cel
+ceva
+chiar
+cînd
+cine
+cineva
+cît
+cîte
+cîţi
+cîtva
+contra
+cu
+cum
+cumva
+curând
+curînd
+da
+dă
+dacă
+dar
+datorită
+de
+deci
+deja
+deoarece
+departe
+deşi
+din
+dinaintea
+dintr
+dintre
+drept
+după
+ea
+ei
+el
+ele
+eram
+este
+eşti
+eu
+face
+fără
+fi
+fie
+fiecare
+fii
+fim
+fiţi
+iar
+ieri
+îi
+îl
+îmi
+împotriva
+în
+înainte
+înaintea
+încât
+încît
+încotro
+între
+întrucât
+întrucît
+îţi
+la
+lângă
+le
+li
+lîngă
+lor
+lui
+mă
+mâine
+mea
+mei
+mele
+mereu
+meu
+mi
+mine
+mult
+multă
+mulţi
+ne
+nicăieri
+nici
+nimeni
+nişte
+noastră
+noastre
+noi
+noştri
+nostru
+nu
+ori
+oricând
+oricare
+oricât
+orice
+oricînd
+oricine
+oricît
+oricum
+oriunde
+până
+pe
+pentru
+peste
+pînă
+poate
+pot
+prea
+prima
+primul
+prin
+printr
+sa
+să
+săi
+sale
+sau
+său
+se
+şi
+sînt
+sîntem
+sînteţi
+spre
+sub
+sunt
+suntem
+sunteţi
+ta
+tăi
+tale
+tău
+te
+ţi
+ţie
+tine
+toată
+toate
+tot
+toţi
+totuşi
+tu
+un
+una
+unde
+undeva
+unei
+unele
+uneori
+unor
+vă
+vi
+voastră
+voastre
+voi
+voştri
+vostru
+vouă
+vreo
+vreun
diff --git a/ckanext/multilingual/solr/schema.xml b/ckanext/multilingual/solr/schema.xml
new file mode 100644
index 0000000..8475187
--- /dev/null
+++ b/ckanext/multilingual/solr/schema.xml
@@ -0,0 +1,452 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="ckan" version="1.3">
+
+<types>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+ <fieldtype name="binary" class="solr.BinaryField"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0"/>
+
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <!-- in this example, we will only use synonyms at query time
+ <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+ -->
+ <!-- Case insensitive stop word removal.
+ add enablePositionIncrements=true in both the index and query
+ analyzers to leave a 'gap' for more accurate phrase queries.
+ -->
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+
+ <!-- A general unstemmed text field - good if one does not know the language of the field -->
+ <fieldType name="textgen" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- English -->
+
+ <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- French -->
+
+ <fieldType name="text_fr" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="french_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.ElisionFilterFactory" articles="fr_elision.txt"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="French" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="french_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.ElisionFilterFactory" articles="fr_elision.txt"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="French" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- German -->
+
+ <fieldType name="text_de" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="german_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="German" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="german_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="German" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Spanish -->
+
+ <fieldType name="text_es" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="spanish_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Spanish" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="spanish_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Spanish" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Italian -->
+
+ <fieldType name="text_it" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="italian_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Italian" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="italian_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Italian" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Dutch -->
+
+ <fieldType name="text_nl" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="dutch_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Dutch" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="dutch_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Dutch" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Romanian -->
+
+ <fieldType name="text_ro" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="romanian_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Romanian" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="romanian_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Romanian" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Portuguese -->
+
+ <fieldType name="text_pt" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="portuguese_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Portuguese" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="portuguese_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Portuguese" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Polish -->
+
+ <fieldType name="text_pl" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="polish_stop.txt" enablePositionIncrements="true" />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="polish_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+</types>
+
+<fields>
+ <field name="index_id" type="string" indexed="true" stored="true" required="true" />
+ <field name="id" type="string" indexed="true" stored="true" required="true" />
+ <field name="site_id" type="string" indexed="true" stored="true" required="true" />
+ <field name="title" type="text" indexed="true" stored="true" />
+ <field name="entity_type" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="state" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="name" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="revision_id" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="version" type="string" indexed="true" stored="true" />
+ <field name="url" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="ckan_url" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="download_url" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="notes" type="text" indexed="true" stored="true"/>
+ <field name="author" type="textgen" indexed="true" stored="true" />
+ <field name="author_email" type="textgen" indexed="true" stored="true" />
+ <field name="maintainer" type="textgen" indexed="true" stored="true" />
+ <field name="maintainer_email" type="textgen" indexed="true" stored="true" />
+ <field name="license" type="string" indexed="true" stored="true" />
+ <field name="license_id" type="string" indexed="true" stored="true" />
+ <field name="ratings_count" type="int" indexed="true" stored="false" />
+ <field name="ratings_average" type="float" indexed="true" stored="false" />
+ <field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
+
+ <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>
+ <field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
+
+ <!-- catchall field, containing all other searchable text fields (implemented
+ via copyField further on in this schema -->
+ <field name="text" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="urls" type="text" indexed="true" stored="false" multiValued="true"/>
+
+ <field name="depends_on" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="dependency_of" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="derives_from" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="has_derivation" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="links_to" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="linked_from" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="child_of" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
+
+ <field name="metadata_created" type="date" indexed="true" stored="true" multiValued="false"/>
+ <field name="metadata_modified" type="date" indexed="true" stored="true" multiValued="false"/>
+
+ <field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
+
+ <!-- Multilingual -->
+ <field name="text_en" type="text_en" indexed="true" stored="true"/>
+ <field name="title_en" type="text_en" indexed="true" stored="true"/>
+
+ <field name="text_fr" type="text_fr" indexed="true" stored="true"/>
+ <field name="title_fr" type="text_fr" indexed="true" stored="true"/>
+
+ <field name="text_de" type="text_de" indexed="true" stored="true"/>
+ <field name="title_de" type="text_de" indexed="true" stored="true"/>
+
+ <field name="text_es" type="text_es" indexed="true" stored="true"/>
+ <field name="title_es" type="text_es" indexed="true" stored="true"/>
+
+ <field name="text_it" type="text_it" indexed="true" stored="true"/>
+ <field name="title_it" type="text_it" indexed="true" stored="true"/>
+
+ <field name="text_nl" type="text_nl" indexed="true" stored="true"/>
+ <field name="title_nl" type="text_nl" indexed="true" stored="true"/>
+
+ <field name="text_ro" type="text_ro" indexed="true" stored="true"/>
+ <field name="title_ro" type="text_ro" indexed="true" stored="true"/>
+
+ <field name="text_pt" type="text_pt" indexed="true" stored="true"/>
+ <field name="title_pt" type="text_pt" indexed="true" stored="true"/>
+
+ <field name="text_pl" type="text_pl" indexed="true" stored="true"/>
+ <field name="title_pl" type="text_pl" indexed="true" stored="true"/>
+
+ <dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
+ <dynamicField name="*" type="string" indexed="true" stored="false"/>
+</fields>
+
+<uniqueKey>index_id</uniqueKey>
+<defaultSearchField>text</defaultSearchField>
+<solrQueryParser defaultOperator="AND"/>
+
+<copyField source="url" dest="urls"/>
+<copyField source="ckan_url" dest="urls"/>
+<copyField source="download_url" dest="urls"/>
+<copyField source="res_url" dest="urls"/>
+<copyField source="extras_*" dest="text"/>
+<copyField source="urls" dest="text"/>
+<copyField source="name" dest="text"/>
+<copyField source="title" dest="text"/>
+<copyField source="text" dest="text"/>
+<copyField source="license" dest="text"/>
+<copyField source="notes" dest="text"/>
+<copyField source="tags" dest="text"/>
+<copyField source="groups" dest="text"/>
+<copyField source="res_description" dest="text"/>
+<copyField source="maintainer" dest="text"/>
+<copyField source="author" dest="text"/>
+
+</schema>
diff --git a/ckanext/multilingual/solr/spanish_stop.txt b/ckanext/multilingual/solr/spanish_stop.txt
new file mode 100644
index 0000000..2db1476
--- /dev/null
+++ b/ckanext/multilingual/solr/spanish_stop.txt
@@ -0,0 +1,354 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A Spanish stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+
+ | The following is a ranked list (commonest to rarest) of stopwords
+ | deriving from a large sample of text.
+
+ | Extra words have been added at the end.
+
+de | from, of
+la | the, her
+que | who, that
+el | the
+en | in
+y | and
+a | to
+los | the, them
+del | de + el
+se | himself, from him etc
+las | the, them
+por | for, by, etc
+un | a
+para | for
+con | with
+no | no
+una | a
+su | his, her
+al | a + el
+ | es from SER
+lo | him
+como | how
+más | more
+pero | pero
+sus | su plural
+le | to him, her
+ya | already
+o | or
+ | fue from SER
+este | this
+ | ha from HABER
+sí | himself etc
+porque | because
+esta | this
+ | son from SER
+entre | between
+ | está from ESTAR
+cuando | when
+muy | very
+sin | without
+sobre | on
+ | ser from SER
+ | tiene from TENER
+también | also
+me | me
+hasta | until
+hay | there is/are
+donde | where
+ | han from HABER
+quien | whom, that
+ | están from ESTAR
+ | estado from ESTAR
+desde | from
+todo | all
+nos | us
+durante | during
+ | estados from ESTAR
+todos | all
+uno | a
+les | to them
+ni | nor
+contra | against
+otros | other
+ | fueron from SER
+ese | that
+eso | that
+ | había from HABER
+ante | before
+ellos | they
+e | and (variant of y)
+esto | this
+mí | me
+antes | before
+algunos | some
+qué | what?
+unos | a
+yo | I
+otro | other
+otras | other
+otra | other
+él | he
+tanto | so much, many
+esa | that
+estos | these
+mucho | much, many
+quienes | who
+nada | nothing
+muchos | many
+cual | who
+ | sea from SER
+poco | few
+ella | she
+estar | to be
+ | haber from HABER
+estas | these
+ | estaba from ESTAR
+ | estamos from ESTAR
+algunas | some
+algo | something
+nosotros | we
+
+ | other forms
+
+mi | me
+mis | mi plural
+tú | thou
+te | thee
+ti | thee
+tu | thy
+tus | tu plural
+ellas | they
+nosotras | we
+vosotros | you
+vosotras | you
+os | you
+mío | mine
+mía |
+míos |
+mías |
+tuyo | thine
+tuya |
+tuyos |
+tuyas |
+suyo | his, hers, theirs
+suya |
+suyos |
+suyas |
+nuestro | ours
+nuestra |
+nuestros |
+nuestras |
+vuestro | yours
+vuestra |
+vuestros |
+vuestras |
+esos | those
+esas | those
+
+ | forms of estar, to be (not including the infinitive):
+estoy
+estás
+está
+estamos
+estáis
+están
+esté
+estés
+estemos
+estéis
+estén
+estaré
+estarás
+estará
+estaremos
+estaréis
+estarán
+estaría
+estarías
+estaríamos
+estaríais
+estarían
+estaba
+estabas
+estábamos
+estabais
+estaban
+estuve
+estuviste
+estuvo
+estuvimos
+estuvisteis
+estuvieron
+estuviera
+estuvieras
+estuviéramos
+estuvierais
+estuvieran
+estuviese
+estuvieses
+estuviésemos
+estuvieseis
+estuviesen
+estando
+estado
+estada
+estados
+estadas
+estad
+
+ | forms of haber, to have (not including the infinitive):
+he
+has
+ha
+hemos
+habéis
+han
+haya
+hayas
+hayamos
+hayáis
+hayan
+habré
+habrás
+habrá
+habremos
+habréis
+habrán
+habría
+habrías
+habríamos
+habríais
+habrían
+había
+habías
+habíamos
+habíais
+habían
+hube
+hubiste
+hubo
+hubimos
+hubisteis
+hubieron
+hubiera
+hubieras
+hubiéramos
+hubierais
+hubieran
+hubiese
+hubieses
+hubiésemos
+hubieseis
+hubiesen
+habiendo
+habido
+habida
+habidos
+habidas
+
+ | forms of ser, to be (not including the infinitive):
+soy
+eres
+es
+somos
+sois
+son
+sea
+seas
+seamos
+seáis
+sean
+seré
+serás
+será
+seremos
+seréis
+serán
+sería
+serías
+seríamos
+seríais
+serían
+era
+eras
+éramos
+erais
+eran
+fui
+fuiste
+fue
+fuimos
+fuisteis
+fueron
+fuera
+fueras
+fuéramos
+fuerais
+fueran
+fuese
+fueses
+fuésemos
+fueseis
+fuesen
+siendo
+sido
+ | sed also means 'thirst'
+
+ | forms of tener, to have (not including the infinitive):
+tengo
+tienes
+tiene
+tenemos
+tenéis
+tienen
+tenga
+tengas
+tengamos
+tengáis
+tengan
+tendré
+tendrás
+tendrá
+tendremos
+tendréis
+tendrán
+tendría
+tendrías
+tendríamos
+tendríais
+tendrían
+tenía
+tenías
+teníamos
+teníais
+tenían
+tuve
+tuviste
+tuvo
+tuvimos
+tuvisteis
+tuvieron
+tuviera
+tuvieras
+tuviéramos
+tuvierais
+tuvieran
+tuviese
+tuvieses
+tuviésemos
+tuvieseis
+tuviesen
+teniendo
+tenido
+tenida
+tenidos
+tenidas
+tened
+
diff --git a/ckanext/multilingual/tests/test_multilingual_plugin.py b/ckanext/multilingual/tests/test_multilingual_plugin.py
new file mode 100644
index 0000000..9a09cc3
--- /dev/null
+++ b/ckanext/multilingual/tests/test_multilingual_plugin.py
@@ -0,0 +1,346 @@
+import ckan.plugins
+import ckanext.multilingual.plugin as mulilingual_plugin
+import ckan.lib.helpers
+import ckan.lib.create_test_data
+import ckan.logic.action.update
+import ckan.tests
+import ckan.tests.html_check
+import routes
+import paste.fixture
+import pylons.test
+import nose
+
+class TestDatasetTermTranslation(ckan.tests.html_check.HtmlCheckMethods):
+ '''Test the translation of datasets by the multilingual_dataset plugin.
+
+ '''
+ @classmethod
+ def setup(cls):
+ cls.app = paste.fixture.TestApp(pylons.test.pylonsapp)
+ ckan.plugins.load('multilingual_dataset')
+ ckan.plugins.load('multilingual_group')
+ ckan.plugins.load('multilingual_tag')
+ ckan.tests.setup_test_search_index()
+ ckan.lib.create_test_data.CreateTestData.create_translations_test_data()
+ # Add translation terms that match a couple of group names and package
+ # names. Group names and package names should _not_ get translated even
+ # if there are terms matching them, because they are used to form URLs.
+ for term in ('roger', 'david', 'annakarenina', 'warandpeace'):
+ for lang_code in ('en', 'de', 'fr'):
+ data_dict = {
+ 'term': term,
+ 'term_translation': 'this should not be rendered',
+ 'lang_code': lang_code,
+ }
+ context = {
+ 'model': ckan.model,
+ 'session': ckan.model.Session,
+ 'user': 'testsysadmin',
+ }
+ ckan.logic.action.update.term_translation_update(context,
+ data_dict)
+
+ @classmethod
+ def teardown(cls):
+ ckan.model.repo.rebuild_db()
+ ckan.lib.search.clear()
+
+ def test_dataset_read_translation(self):
+ '''Test the translation of dataset view pages by the
+ multilingual_dataset plugin.
+
+ '''
+ # Fetch the dataset view page for a number of different languages and
+ # test for the presence of translated and not translated terms.
+ offset = routes.url_for(controller='package', action='read',
+ id='annakarenina')
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ response = self.app.get(offset, status=200,
+ extra_environ={'CKAN_LANG': lang_code,
+ 'CKAN_CURRENT_URL': offset})
+ terms = ('A Novel By Tolstoy',
+ 'Index of the novel',
+ 'russian',
+ 'tolstoy',
+ "Dave's books",
+ "Roger's books",
+ 'Other (Open)',
+ 'romantic novel',
+ 'book',
+ '123',
+ '456',
+ '789',
+ 'plain text',
+ )
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
+ response.mustcontain('<a href="/tag/%s">' % tag_name)
+ for group_name in ('david', 'roger'):
+ response.mustcontain('<a href="/group/%s">' % group_name)
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_tag_read_translation(self):
+ '''Test the translation of tag view pages by the multilingual_tag
+ plugin.
+
+ '''
+ for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
+ offset = routes.url_for(controller='tag', action='read',
+ id=tag_name)
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ response = self.app.get(offset, status=200,
+ extra_environ={'CKAN_LANG': lang_code,
+ 'CKAN_CURRENT_URL': offset})
+ terms = ('A Novel By Tolstoy', tag_name, 'plain text', 'json')
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in (
+ ckan.lib.create_test_data.english_translations):
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[
+ term])
+ else:
+ response.mustcontain(term)
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_user_read_translation(self):
+ '''Test the translation of datasets on user view pages by the
+ multilingual_dataset plugin.
+
+ '''
+ for user_name in ('annafan',):
+ offset = routes.url_for(controller='user', action='read',
+ id=user_name)
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ response = self.app.get(offset, status=200,
+ extra_environ={'CKAN_LANG': lang_code,
+ 'CKAN_CURRENT_URL': offset})
+ terms = ('A Novel By Tolstoy', 'plain text', 'json')
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in (
+ ckan.lib.create_test_data.english_translations):
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[
+ term])
+ else:
+ response.mustcontain(term)
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_group_read_translation(self):
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ offset = '/%s/group/roger' % lang_code
+ response = self.app.get(offset, status=200)
+ terms = ('A Novel By Tolstoy',
+ 'Index of the novel',
+ 'russian',
+ 'tolstoy',
+ #"Dave's books",
+ "Roger's books",
+ #'Other (Open)',
+ #'romantic novel',
+ #'book',
+ '123',
+ '456',
+ '789',
+ 'plain text',
+ 'Roger likes these books.',
+ )
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
+ response.mustcontain('%s?tags=%s' % (offset, tag_name))
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_dataset_index_translation(self):
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ offset = '/%s/dataset' % lang_code
+ response = self.app.get(offset, status=200)
+ for term in ('Index of the novel', 'russian', 'tolstoy',
+ "Dave's books", "Roger's books", 'plain text'):
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
+ response.mustcontain('/%s/dataset?tags=%s' % (lang_code, tag_name))
+ for group_name in ('david', 'roger'):
+ response.mustcontain('/%s/dataset?groups=%s' % (lang_code, group_name))
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_group_index_translation(self):
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ offset = '/%s/group' % lang_code
+ response = self.app.get(offset, status=200)
+ terms = (
+ "Dave's books",
+ "Roger's books",
+ 'Roger likes these books.',
+ "These are books that David likes.",
+ )
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ for group_name in ('david', 'roger'):
+ response.mustcontain('/%s/group/%s' % (lang_code, group_name))
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_tag_index_translation(self):
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ offset = '/%s/tag' % lang_code
+ response = self.app.get(offset, status=200)
+ terms = (
+ "123",
+ "456",
+ '789',
+ "russian",
+ "tolstoy",
+ )
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ response.mustcontain('/%s/tag/%s' % (lang_code, term))
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+class TestDatasetSearchIndex():
+
+ @classmethod
+ def setup_class(cls):
+ ckan.plugins.load('multilingual_dataset')
+ ckan.plugins.load('multilingual_group')
+
+ data_dicts = [
+ {'term': 'moo',
+ 'term_translation': 'french_moo',
+ 'lang_code': 'fr',
+ }, #
+ {'term': 'moo',
+ 'term_translation': 'this should not be rendered',
+ 'lang_code': 'fsdas',
+ },
+ {'term': 'an interesting note',
+ 'term_translation': 'french note',
+ 'lang_code': 'fr',
+ },
+ {'term': 'moon',
+ 'term_translation': 'french moon',
+ 'lang_code': 'fr',
+ },
+ {'term': 'boon',
+ 'term_translation': 'french boon',
+ 'lang_code': 'fr',
+ },
+ {'term': 'boon',
+ 'term_translation': 'italian boon',
+ 'lang_code': 'it',
+ },
+ {'term': 'david',
+ 'term_translation': 'french david',
+ 'lang_code': 'fr',
+ },
+ {'term': 'david',
+ 'term_translation': 'italian david',
+ 'lang_code': 'it',
+ },
+ ]
+
+ context = {
+ 'model': ckan.model,
+ 'session': ckan.model.Session,
+ 'user': 'testsysadmin',
+ 'ignore_auth': True,
+ }
+ for data_dict in data_dicts:
+ ckan.logic.action.update.term_translation_update(context,
+ data_dict)
+
+ def test_translate_terms(self):
+
+ sample_index_data = {
+ 'download_url': u'moo',
+ 'notes': u'an interesting note',
+ 'tags': [u'moon', 'boon'],
+ 'title': u'david',
+ }
+
+ result = mulilingual_plugin.MultilingualDataset().before_index(sample_index_data)
+
+ assert result == {'text_pl': '',
+ 'text_de': '',
+ 'text_ro': '',
+ 'title': u'david',
+ 'notes': u'an interesting note',
+ 'tags': [u'moon', 'boon'],
+ 'title_en': u'david',
+ 'download_url': u'moo',
+ 'text_it': u'italian boon',
+ 'text_es': '',
+ 'text_en': u'an interesting note moon boon moo',
+ 'text_nl': '',
+ 'title_it': u'italian david',
+ 'text_pt': '',
+ 'title_fr': u'french david',
+ 'text_fr': u'french note french boon french_moo french moon'}, result
diff --git a/doc/_themes/sphinx-theme-okfn b/doc/_themes/sphinx-theme-okfn
index 5c2ef73..ef101a1 160000
--- a/doc/_themes/sphinx-theme-okfn
+++ b/doc/_themes/sphinx-theme-okfn
@@ -1 +1 @@
-Subproject commit 5c2ef7385a9e1012d0c328722c8cb235f18391a1
+Subproject commit ef101a18d6de959207361e233b89efd3be24e66f
diff --git a/doc/api.rst b/doc/api.rst
index 18e6823..98d84dc 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -1,4 +1,5 @@
.. index:: API
+.. _api:
========
CKAN API
diff --git a/doc/apiv3.rst b/doc/apiv3.rst
index 22624f6..2ee71ef 100644
--- a/doc/apiv3.rst
+++ b/doc/apiv3.rst
@@ -73,10 +73,14 @@ revision_show id
group_show id
tag_show id
user_show id
+term_translation_show "**terms**" A list of strings, the terms that you want to search for translations of, e.g. "russian", "romantic novel". "**lang_codes**" A list of strings, language codes for the languages that you want to search for translations to, e.g. "en", "de". Optional, if no lang_codes are given translations to all languages will be returned.
package_show_rest id
group_show_rest id
tag_show_rest id
vocabulary_show id
+task_status_show id
+task_status_show entity_id, task_type, key
+resource_status_show id
package_autocomplete q
tag_autocomplete q, fields, offset, limit, vocabulary_id
format_autocomplete q, limit
@@ -122,6 +126,8 @@ group_update_rest (group keys)
user_role_update user OR authorization_group, domain_object, roles
user_role_bulk_update user_roles, domain_object
vocabulary_update (vocabulary keys)
+term_translation_update "**term**" The term that you want to create (or update) a translation for, e.g. "russian", "romantic novel". "**term_translation**" the translation of the term, e.g. "Russisch", "Liebesroman". "**lang_code**" the language code for the translation, e.g. "fr", "de".
+term_translation_update_many "**data**" A list of dictionaries with keys matching the parameter keys for term_translation_update
====================================== ===========================
delete.py:
@@ -205,6 +211,7 @@ key example value notes
======================== ====================================== =============
id "b10871ea-b4ae-4e2e-bec9-a8d8ff357754" (Read-only)
name "country-uk" (Read-only) Add/remove tags from a package or group using update_package or update_group
+display_name "country-uk" (Read-only) display_name is the name of the tag that is displayed to user (as opposed to name which is used to identify the tag, e.g. in URLs). display_name is is usually the same as name but may be different, for example display_names may be translated by the ckanext-multilingual extension.
state "active" (Read-only) Add/remove tags from a package or group using update_package or update_group
revision_timestamp "2009-08-08T12:46:40.920443" (Read-only)
vocabulary_id "Genre" (Read-only) Vocabulary name or id. Optional.
@@ -230,6 +237,16 @@ name "Genre"
tags [{"name":"government-spending"}, {"name": "climate"}] List of tags belonging to this vocabulary.
======================== ===================================================== =============
+Term Translation:
+
+================ ========================= ==================================
+key example value notes
+================ ========================= ==================================
+term "russian" The term that is being translated.
+term_translation "Russisch" The translation of the term.
+lang_code "de" The language of the translation, a language code string.
+================ ========================= ==================================
+
Parameters
==========
diff --git a/doc/datastore.rst b/doc/datastore.rst
index 395de84..15fcd8f 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -30,30 +30,21 @@ the spreadsheet data is stored in the DataStore one would be able to access
individual spreadsheet rows via a simple web-api as well as being able to make
queries over the spreadsheet contents.
-Using the DataStore Data API
-============================
+The DataStore Data API
+======================
The DataStore's Data API, which derives from the underlying ElasticSearch
data-table, is RESTful and JSON-based with extensive query capabilities.
-Each resource in a CKAN instance has an associated DataStore 'database'. This
-database will be accessible via a web interface at::
+Each resource in a CKAN instance has an associated DataStore 'table'. This
+table will be accessible via a web interface at::
/api/data/{resource-id}
This interface to this data is *exactly* the same as that provided by
ElasticSearch to documents of a specific type in one of its indices.
-So, for example, to see the fields in this database do::
-
- /api/data/{resource-id}/_mapping
-
-To do simple search do::
-
- /api/data/{resource-id}/_search?q=abc
-
-For more on searching see: http://www.elasticsearch.org/guide/reference/api/search/uri-request.html
-
+For a detailed tutorial on using this API see :doc:`using-data-api`.
Installation and Configuration
==============================
diff --git a/doc/i18n.rst b/doc/i18n.rst
index 07c2c50..7361525 100644
--- a/doc/i18n.rst
+++ b/doc/i18n.rst
@@ -1,3 +1,5 @@
+.. _i18n:
+
=====================
Internationalize CKAN
=====================
diff --git a/doc/index.rst b/doc/index.rst
index 4fe9aaf..67e1153 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -39,6 +39,7 @@ Customizing and Extending
datastore
background-tasks
geospatial
+ multilingual
Publishing Datasets
===================
@@ -59,6 +60,8 @@ The CKAN API
:maxdepth: 3
api
+ api-tutorial
+ using-data-api
General Administration
======================
diff --git a/doc/multilingual.rst b/doc/multilingual.rst
new file mode 100644
index 0000000..a465b04
--- /dev/null
+++ b/doc/multilingual.rst
@@ -0,0 +1,39 @@
+=====================================
+Translating Datasets, Groups and Tags
+=====================================
+
+For translating CKAN's web interface see :ref:`i18n`. In addition to user interface internationalization, a CKAN administrator can also enter translations into CKAN's database for terms that may appear in the contents of datasets, groups or tags created by users. When a user is viewing the CKAN site, if the translation terms database contains a translation in the user's language for the name or description of a dataset or resource, the name of a tag or group, etc. then the translated term will be shown to the user in place of the original.
+
+Setup and Configuration
+-----------------------
+
+By default term translations are disabled. To enable them, you have to specify the multilingual plugins using the ``ckan.plugins`` setting in your CKAN configuration file, for example:
+
+::
+
+ # List the names of CKAN extensions to activate.
+ ckan.plugins = multilingual_dataset multilingual_group multilingual_tag
+
+Of course, you won't see any terms getting translated until you load some term translations into the database. You can do this using the ``term_translation_update`` and ``term_translation_update_many`` actions of the CKAN API, See :ref:`api` for more details.
+
+Loading Test Translations
+-------------------------
+
+If you want to quickly test the term translation feature without having to provide your own translations, you can load CKAN's test translations into the database by running this command from your shell:
+
+::
+
+ paster --plugin=ckan create-test-data translations
+
+See :ref:`paster` for more details.
+
+Testing The Multilingual Extension
+----------------------------------
+
+If you have a source installation of CKAN you can test the multilingual extension by running the tests located in ``ckanext/multilingual/tests``. You must first install the packages needed for running CKAN tests into your virtual environment, and then run this command from your shell:
+
+::
+
+ nosetests --ckan ckanext/multilingual/tests
+
+See :ref:`basic-tests` for more information.
diff --git a/doc/paster.rst b/doc/paster.rst
index 19d197b..bf94e9f 100644
--- a/doc/paster.rst
+++ b/doc/paster.rst
@@ -1,3 +1,5 @@
+.. _paster:
+
===============================
Common CKAN Administrator Tasks
===============================
diff --git a/doc/post-installation.rst b/doc/post-installation.rst
index 5e39124..5ef039f 100644
--- a/doc/post-installation.rst
+++ b/doc/post-installation.rst
@@ -45,7 +45,13 @@ It can be handy to have some test data to start with. You can get test data like
paster --plugin=ckan create-test-data --config=/etc/ckan/std/std.ini
You now have a CKAN instance that you can log in to, with some test data to check everything
-works.
+works.
+
+You can also create various specialised test data collections for testing specific features of CKAN. For example, ``paster --plugin=ckan create-test-data translations`` creates some test data with some translations for testing the ckanext-multilingual extension. For more information, see:
+
+::
+
+ paster --plugin=ckan create-test-data --help
.. _deployment-notes:
diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst
new file mode 100644
index 0000000..3c6a6fe
--- /dev/null
+++ b/doc/using-data-api.rst
@@ -0,0 +1,402 @@
+==================
+Using the Data API
+==================
+
+Introduction
+============
+
+The Data API builds directly on ElasticSearch, with a resource API endpoint
+being equivalent to a single index 'type' in ElasticSearch (we tend to refer to
+it as a 'table'). This means you can often directly re-use `ElasticSearch
+client libraries`_ when connecting to the API endpoint.
+
+Furthermore, it means that what is presented below is essentially a tutorial in the ElasticSearch API.
+
+.. _ElasticSearch client libraries: http://www.elasticsearch.org/guide/appendix/clients.html
+
+Quickstart
+==========
+
+``endpoint`` refers to the data API endpoint (or ElasticSearch index / table).
+
+Key urls:
+
+* Query: ``{endpoint}/_search`` (in ElasticSearch < 0.19 this will return an
+ error if visited without a query parameter)
+
+ * Query example: ``{endpoint}/_search?size=5&pretty=true``
+
+* Schema (Mapping): ``{endpoint}/_mapping``
+
+Examples
+--------
+
+cURL (or Browser)
+~~~~~~~~~~~~~~~~~
+
+The following examples utilize the <a href="http://curl.haxx.se/">cURL</a>
+command line utility. If you prefer, you you can just open the relevant urls in
+your browser::
+
+ // added pretty=true to get the json results pretty printed
+ curl {endpoint}/_search?q=title:jones&size=5&pretty=true</pre>
+
+Javascript
+~~~~~~~~~~~
+
+A simple ajax (JSONP) request to the data API using jQuery::
+
+ var data = {
+ size: 5 // get 5 results
+ q: 'title:jones' // query on the title field for 'jones'
+ };
+ $.ajax({
+ url: {endpoint}/_search,
+ dataType: 'jsonp',
+ success: function(data) {
+ alert('Total results found: ' + data.hits.total)
+ }
+ });
+
+
+Querying
+========
+
+Basic Queries Using Only the Query String
+-----------------------------------------
+
+Basic queries can be done using only query string parameters in the URL. For
+example, the following searches for text 'hello' in any field in any document
+and returns at most 5 results::
+
+ {endpoint}/_search?q=hello&size=5
+
+Basic queries like this have the advantage that they only involve accessing a
+URL and thus, for example, can be performed just using any web browser.
+However, this method is limited and does not give you access to most of the
+more powerful query features.
+
+Basic queries use the `q` query string parameter which supports the `Lucene
+query parser syntax`_ and hence filters on specific fields (e.g. `fieldname:value`), wildcards (e.g. `abc*`) and more.
+
+.. _Lucene query parser syntax: http://lucene.apache.org/core/old_versioned_docs/versions/3_0_0/queryparsersyntax.html
+
+There are a variety of other options (e.g. size, from etc) that you can also
+specify to customize the query and its results. Full details can be found in
+the `ElasticSearch URI request docs`_.
+
+.. _ElasticSearch URI request docs: http://www.elasticsearch.org/guide/reference/api/search/uri-request.html
+
+Full Query API
+--------------
+
+More powerful and complex queries, including those that involve faceting and
+statistical operations, should use the full ElasticSearch query language and API.
+
+In the query language queries are written as a JSON structure and is then sent
+to the query endpoint (details of the query langague below). There are two
+options for how a query is sent to the search endpoint:
+
+1. Either as the value of a source query parameter e.g.::
+
+ {endpoint}/_search?source={Query-as-JSON}
+
+2. Or in the request body, e.g.::
+
+ curl -XGET {endpoint}/_search -d 'Query-as-JSON'
+
+ For example::
+
+ curl -XGET {endpoint}/_search -d '{
+ "query" : {
+ "term" : { "user": "kimchy" }
+ }
+ }'
+
+
+Query Language
+==============
+
+Queries are JSON objects with the following structure (each of the main
+sections has more detail below)::
+
+ {
+ size: # number of results to return (defaults to 10)
+ from: # offset into results (defaults to 0)
+ fields: # list of document fields that should be returned - http://elasticsearch.org/guide/reference/api/search/fields.html
+ sort: # define sort order - see http://elasticsearch.org/guide/reference/api/search/sort.html
+
+ query: {
+ # "query" object following the Query DSL: http://elasticsearch.org/guide/reference/query-dsl/
+ # details below
+ },
+
+ facets: {
+ # facets specifications
+ # Facets provide summary information about a particular field or fields in the data
+ }
+
+ # special case for situations where you want to apply filter/query to results but *not* to facets
+ filter: {
+ # filter objects
+ # a filter is a simple "filter" (query) on a specific field.
+ # Simple means e.g. checking against a specific value or range of values
+ },
+ }
+
+Query results look like::
+
+ {
+ # some info about the query (which shards it used, how long it took etc)
+ ...
+ # the results
+ hits: {
+ total: # total number of matching documents
+ hits: [
+ # list of "hits" returned
+ {
+ _id: # id of document
+ score: # the search index score
+ _source: {
+ # document 'source' (i.e. the original JSON document you sent to the index
+ }
+ }
+ ]
+ }
+ # facets if these were requested
+ facets: {
+ ...
+ }
+ }
+
+Query DSL: Overview
+-------------------
+
+Query objects are built up of sub-components. These sub-components are either
+basic or compound. Compound sub-components may contains other sub-components
+while basic may not. Example::
+
+ {
+ "query": {
+ # compound component
+ "bool": {
+ # compound component
+ "must": {
+ # basic component
+ "term": {
+ "user": "jones"
+ }
+ }
+ # compound component
+ "must_not": {
+ # basic component
+ "range" : {
+ "age" : {
+ "from" : 10,
+ "to" : 20
+ }
+ }
+ }
+ }
+ }
+ }
+
+In addition, and somewhat confusingly, ElasticSearch distinguishes between
+sub-components that are "queries" and those that are "filters". Filters, are
+really special kind of queries that are: mostly basic (though boolean
+compounding is alllowed); limited to one field or operation and which, as such,
+are especially performant.
+
+Examples, of filters are (full list on RHS at the bottom of the query-dsl_ page):
+
+ * term: filter on a value for a field
+ * range: filter for a field having a range of values (>=, <= etc)
+ * geo_bbox: geo bounding box
+ * geo_distance: geo distance
+
+.. _query-dsl: http://elasticsearch.org/guide/reference/query-dsl/
+
+Rather than attempting to set out all the constraints and options of the
+query-dsl we now offer a variety of examples.
+
+Examples
+--------
+
+Match all / Find Everything
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ {
+ "query": {
+ "match_all": {}
+ }
+ }
+
+Classic Search-Box Style Full-Text Query
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This will perform a full-text style query across all fields. The query string
+supports the `Lucene query parser syntax`_ and hence filters on specific fields
+(e.g. `fieldname:value`), wildcards (e.g. `abc*`) as well as a variety of
+options. For full details see the query-string_ documentation.
+
+::
+
+ {
+ "query": {
+ "query_string": {
+ "query": {query string}
+ }
+ }
+ }
+
+.. _query-string: http://elasticsearch.org/guide/reference/query-dsl/query-string-query.html
+
+Filter on One Field
+~~~~~~~~~~~~~~~~~~~
+
+::
+
+ {
+ "query": {
+ "term": {
+ {field-name}: {value}
+ }
+ }
+ }
+
+High performance equivalent using filters::
+
+ {
+ "query": {
+ "constant_score": {
+ "filter": {
+ "term": {
+ # note that value should be *lower-cased*
+ {field-name}: {value}
+ }
+ }
+ }
+ }
+
+Find all documents with value in a range
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This can be used both for text ranges (e.g. A to Z), numeric ranges (10-20) and
+for dates (ElasticSearch will converts dates to ISO 8601 format so you can
+search as 1900-01-01 to 1920-02-03).
+
+::
+
+ {
+ "query": {
+ "constant_score": {
+ "filter": {
+ "range": {
+ {field-name}: {
+ "from": {lower-value}
+ "to": {upper-value}
+ }
+ }
+ }
+ }
+ }
+ }
+
+For more details see `range filters`_.
+
+.. _range filters: http://elasticsearch.org/guide/reference/query-dsl/range-filter.html
+
+Full-Text Query plus Filter on a Field
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ {
+ "query": {
+ "query_string": {
+ "query": {query string}
+ },
+ "term": {
+ {field}: {value}
+ }
+ }
+ }
+
+
+Filter on two fields
+~~~~~~~~~~~~~~~~~~~~
+
+Note that you cannot, unfortunately, have a simple and query by adding two
+filters inside the query element. Instead you need an 'and' clause in a filter
+(which in turn requires nesting in 'filtered'). You could also achieve the same
+result here using a `bool query`_.
+
+.. _bool query: http://elasticsearch.org/guide/reference/query-dsl/bool-query.html
+
+::
+
+ {
+ "query": {
+ "filtered": {
+ "query": {
+ "match_all": {}
+ },
+ "filter": {
+ "and": [
+ {
+ "range" : {
+ "b" : {
+ "from" : 4,
+ "to" : "8"
+ }
+ },
+ },
+ {
+ "term": {
+ "a": "john"
+ }
+ }
+ ]
+ }
+ }
+ }
+ }
+
+Facets
+------
+
+Facets provide a way to get summary information about then data in an
+elasticsearch table, for example counts of distinct values.
+
+TODO: complete
+
+
+Schema Mapping
+==============
+
+As the ElasticSearch documentation states:
+
+ Mapping is the process of defining how a document should be mapped to the
+ Search Engine, including its searchable characteristics such as which fields
+ are searchable and if/how they are tokenized. In ElasticSearch, an index may
+ store documents of different “mapping types”. ElasticSearch allows one to
+ associate multiple mapping definitions for each mapping type.
+
+ Explicit mapping is defined on an index/type level. By default, there isn't a
+ need to define an explicit mapping, since one is automatically created and
+ registered when a new type or new field is introduced (with no performance
+ overhead) and have sensible defaults. Only when the defaults need to be
+ overridden must a mapping definition be provided.
+
+Relevant docs: http://elasticsearch.org/guide/reference/mapping/.
+
+
+JSONP support
+=============
+
+JSONP support is available on any request via a simple callback query string parameter::
+
+ ?callback=my_callback_name
+
diff --git a/setup.py b/setup.py
index ee1f06d..33f6d6b 100644
--- a/setup.py
+++ b/setup.py
@@ -91,6 +91,11 @@
[ckan.plugins]
synchronous_search = ckan.lib.search:SynchronousSearchPlugin
stats=ckanext.stats.plugin:StatsPlugin
+ publisher_form=ckanext.publisher_form.forms:PublisherForm
+ publisher_dataset_form=ckanext.publisher_form.forms:PublisherDatasetForm
+ multilingual_dataset=ckanext.multilingual.plugin:MultilingualDataset
+ multilingual_group=ckanext.multilingual.plugin:MultilingualGroup
+ multilingual_tag=ckanext.multilingual.plugin:MultilingualTag
organizations=ckanext.organizations.forms:OrganizationForm
organizations_dataset=ckanext.organizations.forms:OrganizationDatasetForm
test_tag_vocab_plugin=ckanext.test_tag_vocab_plugin:MockVocabTagsPlugin
================================================================
Compare: https://github.com/okfn/ckan/compare/78deae0...fbf7754
More information about the ckan-changes
mailing list