[ckan-changes] [okfn/ckan] 31084c: Merge branch 'master' of https://github.com/okfn/c...
GitHub
noreply at github.com
Tue Apr 24 15:01:10 UTC 2012
Branch: refs/heads/master
Home: https://github.com/okfn/ckan
Commit: 31084c9bf0dffc02f4b6996ea15e26860f485f4c
https://github.com/okfn/ckan/commit/31084c9bf0dffc02f4b6996ea15e26860f485f4c
Author: Ross Jones <rossdjones at gmail.com>
Date: 2012-04-19 (Thu, 19 Apr 2012)
Changed paths:
M ckan/lib/base.py
M ckan/lib/helpers.py
M ckan/plugins/__init__.py
A ckan/plugins/toolkit.py
Log Message:
-----------
Merge branch 'master' of https://github.com/okfn/ckan
diff --git a/ckan/lib/base.py b/ckan/lib/base.py
index 30914ba..da9d69f 100644
--- a/ckan/lib/base.py
+++ b/ckan/lib/base.py
@@ -47,9 +47,29 @@ def abort(status_code=None, detail='', headers=None, comment=None):
headers=headers,
comment=comment)
+
+def render_snippet(template_name, **kw):
+ ''' Helper function for rendering snippets. Rendered html has
+ comment tags added to show the template used. NOTE: unlike other
+ render functions this takes a list of keywords instead of a dict for
+ the extra template variables. '''
+ output = render(template_name, extra_vars=kw)
+ output = '\n<!-- Snippet %s start -->\n%s\n<!-- Snippet %s end -->\n' % (
+ template_name, output, template_name)
+ return literal(output)
+
+def render_text(template_name, extra_vars=None):
+ ''' Helper function to render a genshi NewTextTemplate without
+ having to pass the loader_class or method. '''
+ return render(template_name,
+ extra_vars=extra_vars,
+ method='text',
+ loader_class=NewTextTemplate)
+
def render(template_name, extra_vars=None, cache_key=None, cache_type=None,
cache_expire=None, method='xhtml', loader_class=MarkupTemplate,
cache_force = None):
+ ''' Main genshi template rendering function. '''
def render_template():
globs = extra_vars or {}
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index b0fefe1..b4dd06e 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -31,9 +31,6 @@
from pylons import session
from pylons import c
from pylons.i18n import _
-from pylons.templating import pylons_globals
-from genshi.template import MarkupTemplate
-from ckan.plugins import PluginImplementations, IGenshiStreamFilter
get_available_locales = i18n.get_available_locales
get_locales_dict = i18n.get_locales_dict
@@ -675,20 +672,8 @@ def activity_div(template, activity, actor, object=None, target=None):
def snippet(template_name, **kw):
''' This function is used to load html snippets into pages. keywords
can be used to pass parameters into the snippet rendering '''
- pylons_globs = pylons_globals()
- genshi_loader = pylons_globs['app_globals'].genshi_loader
- template = genshi_loader.load(template_name, cls=MarkupTemplate)
- globs = kw
- globs['h'] = pylons_globs['h']
- globs['c'] = pylons_globs['c']
- globs['config'] = pylons_globs['config']
- stream = template.generate(**globs)
- for item in PluginImplementations(IGenshiStreamFilter):
- stream = item.filter(stream)
- output = stream.render(method='xhtml', encoding=None, strip_whitespace=True)
- output = '\n<!-- Snippet %s start -->\n%s\n<!-- Snippet %s end -->\n' % (
- template_name, output, template_name)
- return literal(output)
+ import ckan.lib.base as base
+ return base.render_snippet(template_name, **kw)
def convert_to_dict(object_type, objs):
diff --git a/ckan/plugins/__init__.py b/ckan/plugins/__init__.py
index 3784b59..fcb6055 100644
--- a/ckan/plugins/__init__.py
+++ b/ckan/plugins/__init__.py
@@ -1,2 +1,3 @@
from ckan.plugins.core import *
from ckan.plugins.interfaces import *
+import toolkit
diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py
new file mode 100644
index 0000000..64a80d7
--- /dev/null
+++ b/ckan/plugins/toolkit.py
@@ -0,0 +1,75 @@
+## This file is intended to make functions consistently available to
+## plugins whilst giving developers the ability move code around or
+## change underlying frameworks etc. It should not be used internaly
+## within ckan only by extensions. Functions should only be removed from
+## this file after reasonable depreciation notice has been given.
+
+import inspect
+import os
+
+import pylons
+import paste.deploy.converters as converters
+import webhelpers.html.tags
+
+import lib.base as base
+
+
+__all__ = [
+ ## Imported functions ##
+ 'c', # template context
+ 'request', # http request object
+ 'render', # template render function
+ 'render_text', # Genshi NewTextTemplate render function
+ 'render_snippet', # snippet render function
+ 'asbool', # converts an object to a boolean
+ 'asint', # converts an object to an integer
+ 'aslist', # converts an object to a list
+ 'literal', # stop tags in a string being escaped
+
+ ## Functions fully defined here ##
+ 'add_template_directory',
+ 'add_public_directory',
+]
+
+c = pylons.c
+request = pylons.request
+render = base.render
+render_text = base.render_text
+asbool = converters.asbool
+asint = converters.asint
+aslist = converters.aslist
+literal = webhelpers.html.tags.literal
+
+
+# wrappers
+def render_snippet(template, data=None):
+ data = data or {}
+ return base.render_snippet(template, **data)
+
+
+# new functions
+def add_template_directory(config, relative_path):
+ ''' Function to aid adding extra template paths to the config.
+ The path is relative to the file calling this function. '''
+ _add_served_directory(config, relative_path, 'extra_template_paths')
+
+def add_public_directory(config, relative_path):
+ ''' Function to aid adding extra public paths to the config.
+ The path is relative to the file calling this function. '''
+ _add_served_directory(config, relative_path, 'extra_public_paths')
+
+def _add_served_directory(config, relative_path, config_var):
+ ''' Add extra public/template directories to config. '''
+ assert config_var in ('extra_template_paths', 'extra_public_paths')
+ # we want the filename that of the function caller but they will
+ # have used one of the available helper functions
+ frame, filename, line_number, function_name, lines, index =\
+ inspect.getouterframes(inspect.currentframe())[2]
+
+ this_dir = os.path.dirname(filename)
+ absolute_path = os.path.join(this_dir, relative_path)
+ if absolute_path not in config.get(config_var, ''):
+ if config.get(config_var):
+ config[config_var] += ',' + absolute_path
+ else:
+ config[config_var] = absolute_path
================================================================
Commit: 762901f7d21dbaecf3f120c21d9136375c632a77
https://github.com/okfn/ckan/commit/762901f7d21dbaecf3f120c21d9136375c632a77
Author: Ross Jones <rossdjones at gmail.com>
Date: 2012-04-20 (Fri, 20 Apr 2012)
Changed paths:
M ckan/lib/base.py
M ckan/lib/cli.py
M ckan/plugins/__init__.py
M ckan/plugins/toolkit.py
Log Message:
-----------
Merge branch 'master' of https://github.com/okfn/ckan
diff --git a/ckan/lib/base.py b/ckan/lib/base.py
index da9d69f..075029c 100644
--- a/ckan/lib/base.py
+++ b/ckan/lib/base.py
@@ -22,7 +22,7 @@
import ckan.exceptions
import ckan
-from ckan import authz
+import ckan.authz as authz
from ckan.lib import i18n
import ckan.lib.helpers as h
from ckan.plugins import PluginImplementations, IGenshiStreamFilter
diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
index 5146aa9..6798cc4 100644
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -84,7 +84,7 @@ class ManageDb(CkanCommand):
def command(self):
self._load_config()
- from ckan import model
+ import ckan.model as model
import ckan.lib.search as search
cmd = self.args[0]
@@ -164,7 +164,7 @@ def _postgres_dump(self, filepath):
self._run_cmd(pg_dump_cmd)
def _postgres_load(self, filepath):
- from ckan import model
+ import ckan.model as model
assert not model.repo.are_tables_created(), "Tables already found. You need to 'db clean' before a load."
pg_cmd = self._get_psql_cmd() + ' -f %s' % filepath
self._run_cmd(pg_cmd)
@@ -194,7 +194,7 @@ def load(self, only_load=False):
pg_cmd = self._postgres_load(dump_path)
if not only_load:
print 'Upgrading DB'
- from ckan import model
+ import ckan.model as model
model.repo.upgrade_db()
print 'Rebuilding search index'
@@ -205,7 +205,7 @@ def load(self, only_load=False):
print 'Done'
def simple_dump_csv(self):
- from ckan import model
+ import ckan.model as model
if len(self.args) < 2:
print 'Need csv file path'
return
@@ -215,7 +215,7 @@ def simple_dump_csv(self):
dumper.SimpleDumper().dump(dump_file, format='csv')
def simple_dump_json(self):
- from ckan import model
+ import ckan.model as model
if len(self.args) < 2:
print 'Need json file path'
return
@@ -452,7 +452,7 @@ class Sysadmin(CkanCommand):
def command(self):
self._load_config()
- from ckan import model
+ import ckan.model as model
cmd = self.args[0] if self.args else None
if cmd == None or cmd == 'list':
@@ -465,7 +465,7 @@ def command(self):
print 'Command %s not recognized' % cmd
def list(self):
- from ckan import model
+ import ckan.model as model
print 'Sysadmins:'
sysadmins = model.Session.query(model.SystemRole).filter_by(role=model.Role.ADMIN)
print 'count = %i' % sysadmins.count()
@@ -477,7 +477,7 @@ def list(self):
user_or_authgroup.id)
def add(self):
- from ckan import model
+ import ckan.model as model
if len(self.args) < 2:
print 'Need name of the user to be made sysadmin.'
@@ -501,7 +501,7 @@ def add(self):
print 'Added %s as sysadmin' % username
def remove(self):
- from ckan import model
+ import ckan.model as model
if len(self.args) < 2:
print 'Need name of the user to be made sysadmin.'
@@ -537,7 +537,7 @@ class UserCmd(CkanCommand):
def command(self):
self._load_config()
- from ckan import model
+ import ckan.model as model
if not self.args:
self.list()
@@ -563,7 +563,7 @@ def get_user_str(self, user):
return user_str
def list(self):
- from ckan import model
+ import ckan.model as model
print 'Users:'
users = model.Session.query(model.User)
print 'count = %i' % users.count()
@@ -571,14 +571,14 @@ def list(self):
print self.get_user_str(user)
def show(self):
- from ckan import model
+ import ckan.model as model
username = self.args[0]
user = model.User.get(unicode(username))
print 'User: \n', user
def setpass(self):
- from ckan import model
+ import ckan.model as model
if len(self.args) < 2:
print 'Need name of the user.'
@@ -593,7 +593,7 @@ def setpass(self):
print 'Done'
def search(self):
- from ckan import model
+ import ckan.model as model
if len(self.args) < 2:
print 'Need user name query string.'
@@ -618,7 +618,7 @@ def password_prompt(cls):
return password1
def add(self):
- from ckan import model
+ import ckan.model as model
if len(self.args) < 2:
print 'Need name of the user.'
@@ -671,7 +671,7 @@ def add(self):
print user
def remove(self):
- from ckan import model
+ import ckan.model as model
if len(self.args) < 2:
print 'Need name of the user.'
@@ -704,7 +704,7 @@ class DatasetCmd(CkanCommand):
def command(self):
self._load_config()
- from ckan import model
+ import ckan.model as model
if not self.args:
print self.usage
@@ -722,7 +722,7 @@ def command(self):
self.show(self.args[0])
def list(self):
- from ckan import model
+ import ckan.model as model
print 'Datasets:'
datasets = model.Session.query(model.Package)
print 'count = %i' % datasets.count()
@@ -732,19 +732,19 @@ def list(self):
print '%s %s %s' % (dataset.id, dataset.name, state)
def _get_dataset(self, dataset_ref):
- from ckan import model
+ import ckan.model as model
dataset = model.Package.get(unicode(dataset_ref))
assert dataset, 'Could not find dataset matching reference: %r' % dataset_ref
return dataset
def show(self, dataset_ref):
- from ckan import model
import pprint
dataset = self._get_dataset(dataset_ref)
pprint.pprint(dataset.as_dict())
def delete(self, dataset_ref):
- from ckan import model, plugins
+ from ckan import plugins
+ import ckan.model as model
dataset = self._get_dataset(dataset_ref)
old_state = dataset.state
@@ -756,7 +756,8 @@ def delete(self, dataset_ref):
print '%s %s -> %s' % (dataset.name, old_state, dataset.state)
def purge(self, dataset_ref):
- from ckan import model, plugins
+ from ckan import plugins
+ import ckan.model as model
dataset = self._get_dataset(dataset_ref)
name = dataset.name
@@ -802,7 +803,7 @@ def run_(self):
def view(self):
self._load_config()
- from ckan import model
+ import ckan.model as model
from kombu.transport.sqlalchemy.models import Message
q = model.Session.query(Message)
q_visible = q.filter_by(visible=True)
@@ -816,7 +817,7 @@ def view(self):
def clean(self):
self._load_config()
- from ckan import model
+ import ckan.model as model
import pprint
tasks_initially = model.Session.execute("select * from kombu_message").rowcount
if not tasks_initially:
@@ -847,7 +848,7 @@ class Ratings(CkanCommand):
def command(self):
self._load_config()
- from ckan import model
+ import ckan.model as model
cmd = self.args[0]
if cmd == 'count':
@@ -860,14 +861,14 @@ def command(self):
print 'Command %s not recognized' % cmd
def count(self):
- from ckan import model
+ import ckan.model as model
q = model.Session.query(model.Rating)
print "%i ratings" % q.count()
q = q.filter(model.Rating.user_id == None)
print "of which %i are anonymous ratings" % q.count()
def clean(self, user_ratings=True):
- from ckan import model
+ import ckan.model as model
q = model.Session.query(model.Rating)
print "%i ratings" % q.count()
if not user_ratings:
diff --git a/ckan/plugins/__init__.py b/ckan/plugins/__init__.py
index fcb6055..e11e909 100644
--- a/ckan/plugins/__init__.py
+++ b/ckan/plugins/__init__.py
@@ -1,3 +1,19 @@
from ckan.plugins.core import *
from ckan.plugins.interfaces import *
-import toolkit
+
+
+class _Toolkit(object):
+ ''' This object allows us to avoid circular imports while making
+ functions/objects available to plugins. '''
+
+ def __init__(self):
+ self.toolkit = None
+
+ def __getattr__(self, name):
+ if not self.toolkit:
+ import toolkit
+ self.toolkit = toolkit
+ return getattr(self.toolkit, name)
+
+toolkit = _Toolkit()
+del _Toolkit
diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py
index 64a80d7..e635c43 100644
--- a/ckan/plugins/toolkit.py
+++ b/ckan/plugins/toolkit.py
@@ -1,21 +1,28 @@
-## This file is intended to make functions consistently available to
-## plugins whilst giving developers the ability move code around or
-## change underlying frameworks etc. It should not be used internaly
-## within ckan only by extensions. Functions should only be removed from
-## this file after reasonable depreciation notice has been given.
+## This file is intended to make functions/objects consistently
+## available to plugins whilst giving developers the ability move code
+## around or change underlying frameworks etc. It should not be used
+## internally within ckan only by extensions. Functions should only be
+## removed from this file after reasonable depreciation notice has
+## been given.
import inspect
import os
+import re
import pylons
import paste.deploy.converters as converters
import webhelpers.html.tags
-import lib.base as base
+import ckan
+import ckan.lib.base as base
+import ckan.logic as logic
+import ckan.lib.cli as cli
+
__all__ = [
- ## Imported functions ##
+ ## Imported functions/objects ##
+ '_', # i18n translation
'c', # template context
'request', # http request object
'render', # template render function
@@ -25,12 +32,22 @@
'asint', # converts an object to an integer
'aslist', # converts an object to a list
'literal', # stop tags in a string being escaped
+ 'get_action', # get logic action function
+ 'check_access', # check logic function authorisation
+ 'ActionNotFound', # action not found exception (ckan.logic.NotFound)
+ 'NotAuthorized', # action not authorized exception
+ 'ValidationError', # model update validation error
+ 'CkanCommand', # class for providing cli interfaces
- ## Functions fully defined here ##
+ ## Fully defined in this file ##
'add_template_directory',
'add_public_directory',
+ 'requires_ckan_version',
+ 'check_ckan_version',
+ 'CkanVersionException',
]
+_ = pylons.i18n._
c = pylons.c
request = pylons.request
render = base.render
@@ -40,6 +57,13 @@
aslist = converters.aslist
literal = webhelpers.html.tags.literal
+get_action = logic.get_action
+check_access = logic.check_access
+ActionNotFound = logic.NotFound ## Name change intentional
+NotAuthorized = logic.NotAuthorized
+ValidationError = logic.ValidationError
+
+CkanCommand = cli.CkanCommand
# wrappers
def render_snippet(template, data=None):
@@ -73,3 +97,38 @@ def _add_served_directory(config, relative_path, config_var):
config[config_var] += ',' + absolute_path
else:
config[config_var] = absolute_path
+
+class CkanVersionException(Exception):
+ ''' Exception raised if required ckan version is not available. '''
+ pass
+
+
+def _version_str_2_list(v_str):
+ ''' conver a version string into a list of ints
+ eg 1.6.1b --> [1, 6, 1] '''
+ v_str = re.sub(r'[^0-9.]', '', v_str)
+ return [int(part) for part in v_str.split('.')]
+
+def check_ckan_version(min_version=None, max_version=None):
+ ''' Check that the ckan version is correct for the plugin. '''
+ current = _version_str_2_list(ckan.__version__)
+
+ if min_version:
+ min_required = _version_str_2_list(min_version)
+ if current < min_required:
+ return False
+ if max_version:
+ max_required = _version_str_2_list(max_version)
+ if current > max_required:
+ return False
+ return True
+
+def requires_ckan_version(min_version, max_version=None):
+ ''' Check that the ckan version is correct for the plugin. '''
+ if not check_ckan_version(min_version=min_version, max_version=max_version):
+ if not max_version:
+ error = 'Requires ckan version %s or higher' % min_version
+ else:
+ error = 'Requires ckan version between %s and %s' % \
+ (min_version, max_version)
+ raise CkanVersionException(error)
================================================================
Commit: f7b06d034faf0897334d42f917fe942be3e19530
https://github.com/okfn/ckan/commit/f7b06d034faf0897334d42f917fe942be3e19530
Author: Ross Jones <rossdjones at gmail.com>
Date: 2012-04-23 (Mon, 23 Apr 2012)
Changed paths:
M ckan/ckan_nose_plugin.py
M ckan/config/environment.py
M ckan/controllers/group.py
M ckan/controllers/package.py
M ckan/controllers/tag.py
M ckan/controllers/user.py
M ckan/lib/base.py
M ckan/lib/celery_app.py
M ckan/lib/create_test_data.py
M ckan/lib/dictization/model_dictize.py
M ckan/lib/helpers.py
R ckan/lib/helpers_clean.py
M ckan/logic/__init__.py
M ckan/logic/action/get.py
M ckan/logic/auth/get.py
M ckan/logic/auth/publisher/get.py
M ckan/logic/schema.py
M ckan/model/__init__.py
M ckan/plugins/__init__.py
M ckan/plugins/interfaces.py
M ckan/plugins/toolkit.py
M ckan/public/scripts/application.js
M ckan/templates/_snippet/data-api-help.html
M ckan/templates/_util.html
M ckan/templates/facets.html
M ckan/templates/group/read.html
M ckan/templates/package/search.html
M ckan/templates/tag/index.html
M ckan/templates/tag/read.html
M ckan/tests/functional/test_cors.py
M ckan/tests/lib/test_dictization.py
A ckan/tests/logic/__init__.py
M ckan/tests/logic/test_action.py
A ckanext/multilingual/plugin.py
A ckanext/multilingual/solr/dutch_stop.txt
A ckanext/multilingual/solr/english_stop.txt
A ckanext/multilingual/solr/fr_elision.txt
A ckanext/multilingual/solr/french_stop.txt
A ckanext/multilingual/solr/german_stop.txt
A ckanext/multilingual/solr/greek_stopwords.txt
A ckanext/multilingual/solr/italian_stop.txt
A ckanext/multilingual/solr/polish_stop.txt
A ckanext/multilingual/solr/portuguese_stop.txt
A ckanext/multilingual/solr/romanian_stop.txt
A ckanext/multilingual/solr/schema.xml
A ckanext/multilingual/solr/spanish_stop.txt
A ckanext/multilingual/tests/test_multilingual_plugin.py
M doc/_themes/sphinx-theme-okfn
M doc/api.rst
M doc/apiv3.rst
M doc/datastore.rst
M doc/i18n.rst
M doc/index.rst
A doc/multilingual.rst
M doc/paster.rst
M doc/post-installation.rst
A doc/using-data-api.rst
M setup.py
Log Message:
-----------
Merge branch 'master' of https://github.com/okfn/ckan
diff --git a/ckan/ckan_nose_plugin.py b/ckan/ckan_nose_plugin.py
index 7018e31..bc995db 100644
--- a/ckan/ckan_nose_plugin.py
+++ b/ckan/ckan_nose_plugin.py
@@ -30,7 +30,6 @@ def startContext(self, ctx):
# init_db is run at the start of every class because
# when you use an in-memory sqlite db, it appears that
# the db is destroyed after every test when you Session.Remove().
- model.repo.init_db()
## This is to make sure the configuration is run again.
## Plugins use configure to make their own tables and they
@@ -40,6 +39,7 @@ def startContext(self, ctx):
for plugin in PluginImplementations(IConfigurable):
plugin.configure(config)
+ model.repo.init_db()
def options(self, parser, env):
parser.add_option(
diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index fa91541..8d42f4b 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -1,31 +1,85 @@
"""Pylons environment configuration"""
import os
-from urlparse import urlparse
import logging
import warnings
-
-from paste.deploy.converters import asbool
-
-# Suppress benign warning 'Unbuilt egg for setuptools'
-warnings.simplefilter('ignore', UserWarning)
-
+from urlparse import urlparse
import pylons
+from paste.deploy.converters import asbool
import sqlalchemy
-
from pylons import config
-from pylons.i18n.translation import ugettext
from genshi.template import TemplateLoader
from genshi.filters.i18n import Translator
-from paste.deploy.converters import asbool
-import ckan.lib.app_globals as app_globals
+import ckan.config.routing as routing
+import ckan.model as model
+import ckan.plugins as p
import ckan.lib.helpers as h
-from ckan.config.routing import make_map
-from ckan import model
-from ckan import plugins
+import ckan.lib.search as search
+import ckan.lib.app_globals as app_globals
+# Suppress benign warning 'Unbuilt egg for setuptools'
+warnings.simplefilter('ignore', UserWarning)
+
+class _Helpers(object):
+ ''' Helper object giving access to template helpers stopping
+ missing functions from causing template exceptions. Useful if
+ templates have helper functions provided by extensions that have
+ not been enabled. '''
+ def __init__(self, helpers, restrict=True):
+ functions = {}
+ allowed = helpers.__allowed_functions__
+ # list of functions due to be depreciated
+ self.depreciated = []
+
+ for helper in dir(helpers):
+ if helper not in allowed:
+ self.depreciated.append(helper)
+ if restrict:
+ continue
+ functions[helper] = getattr(helpers, helper)
+ self.functions = functions
+
+ # extend helper functions with ones supplied by plugins
+ extra_helpers = []
+ for plugin in p.PluginImplementations(p.ITemplateHelpers):
+ helpers = plugin.get_helpers()
+ for helper in helpers:
+ if helper in extra_helpers:
+ raise Exception('overwritting extra helper %s' % helper)
+ extra_helpers.append(helper)
+ functions[helper] = helpers[helper]
+ # logging
+ self.log = logging.getLogger('ckan.helpers')
+
+ @classmethod
+ def null_function(cls, *args, **kw):
+ ''' This function is returned if no helper is found. The idea is
+ to try to allow templates to be rendered even if helpers are
+ missing. Returning the empty string seems to work well.'''
+ return ''
+
+ def __getattr__(self, name):
+ ''' return the function/object requested '''
+ if name in self.functions:
+ if name in self.depreciated:
+ msg = 'Template helper function `%s` is depriciated' % name
+ self.log.warn(msg)
+ return self.functions[name]
+ else:
+ if name in self.depreciated:
+ msg = 'Template helper function `%s` is not available ' \
+ 'as it has been depriciated.\nYou can enable it ' \
+ 'by setting ckan.restrict_template_vars = true ' \
+ 'in your .ini file.' % name
+ self.log.critical(msg)
+ else:
+ msg = 'Helper function `%s` could not be found\n ' \
+ '(are you missing an extension?)' % name
+ self.log.critical(msg)
+ return self.null_function
+
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
@@ -65,12 +119,9 @@ def find_controller(self, controller):
config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# load all CKAN plugins
- plugins.load_all(config)
+ p.load_all(config)
- from ckan.plugins import PluginImplementations
- from ckan.plugins.interfaces import IConfigurer
-
- for plugin in PluginImplementations(IConfigurer):
+ for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
@@ -90,33 +141,19 @@ def find_controller(self, controller):
config['ckan.site_id'] = ckan_host
# Init SOLR settings and check if the schema is compatible
- from ckan.lib.search import SolrSettings, check_solr_schema_version
- SolrSettings.init(config.get('solr_url'),
- config.get('solr_user'),
- config.get('solr_password'))
- check_solr_schema_version()
+ #from ckan.lib.search import SolrSettings, check_solr_schema_version
+ search.SolrSettings.init(config.get('solr_url'),
+ config.get('solr_user'),
+ config.get('solr_password'))
+ search.check_solr_schema_version()
- config['routes.map'] = make_map()
+ config['routes.map'] = routing.make_map()
config['pylons.app_globals'] = app_globals.Globals()
- if asbool(config.get('ckan.restrict_template_vars', 'false')):
- import ckan.lib.helpers_clean
- config['pylons.h'] = ckan.lib.helpers_clean
- else:
- config['pylons.h'] = h
-
- # extend helper functions with ones supplied by plugins
- from ckan.plugins import PluginImplementations
- from ckan.plugins.interfaces import ITemplateHelpers
-
- extra_helpers = []
- for plugin in PluginImplementations(ITemplateHelpers):
- helpers = plugin.get_helpers()
- for helper in helpers:
- if helper in extra_helpers:
- raise Exception('overwritting extra helper %s' % helper)
- extra_helpers.append(helper)
- setattr(config['pylons.h'], helper, helpers[helper])
+ # add helper functions
+ restrict_helpers = asbool(config.get('ckan.restrict_template_vars', 'false'))
+ helpers = _Helpers(h, restrict_helpers)
+ config['pylons.h'] = helpers
## redo template setup to use genshi.search_path (so remove std template setup)
template_paths = [paths['templates'][0]]
@@ -151,16 +188,12 @@ def template_loaded(template):
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
- engine = sqlalchemy.create_engine(ckan_db)
- else:
- engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
+ config['sqlalchemy.url'] = ckan_db
+ engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
if not model.meta.engine:
model.init_model(engine)
- from ckan.plugins import PluginImplementations
- from ckan.plugins.interfaces import IConfigurable
-
- for plugin in PluginImplementations(IConfigurable):
+ for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index 4e7cda3..95a4597 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -72,7 +72,7 @@ def index(self):
group_type = self._guess_group_type()
context = {'model': model, 'session': model.Session,
- 'user': c.user or c.author}
+ 'user': c.user or c.author, 'for_view': True}
data_dict = {'all_fields': True}
@@ -194,6 +194,7 @@ def pager_url(q=None, page=None):
items_per_page=limit
)
c.facets = query['facets']
+ c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Group search error: %r', se.args)
diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py
index e90d658..18c2b2a 100644
--- a/ckan/controllers/package.py
+++ b/ckan/controllers/package.py
@@ -210,6 +210,7 @@ def pager_url(q=None, page=None):
items_per_page=limit
)
c.facets = query['facets']
+ c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
diff --git a/ckan/controllers/tag.py b/ckan/controllers/tag.py
index f623f05..06bccd7 100644
--- a/ckan/controllers/tag.py
+++ b/ckan/controllers/tag.py
@@ -25,9 +25,9 @@ def index(self):
c.q = request.params.get('q', '')
context = {'model': model, 'session': model.Session,
- 'user': c.user or c.author}
+ 'user': c.user or c.author, 'for_view': True}
- data_dict = {}
+ data_dict = {'all_fields': True}
if c.q:
page = int(request.params.get('page', 1))
@@ -58,7 +58,7 @@ def index(self):
def read(self, id):
context = {'model': model, 'session': model.Session,
- 'user': c.user or c.author}
+ 'user': c.user or c.author, 'for_view': True}
data_dict = {'id':id}
try:
diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py
index 0a26d59..7f46b4f 100644
--- a/ckan/controllers/user.py
+++ b/ckan/controllers/user.py
@@ -83,7 +83,7 @@ def index(self):
def read(self, id=None):
context = {'model': model,
- 'user': c.user or c.author}
+ 'user': c.user or c.author, 'for_view': True}
data_dict = {'id':id,
'user_obj':c.userobj}
try:
diff --git a/ckan/lib/base.py b/ckan/lib/base.py
index 075029c..dd1f03b 100644
--- a/ckan/lib/base.py
+++ b/ckan/lib/base.py
@@ -245,8 +245,8 @@ def __after__(self, action, **params):
def _set_cors(self):
response.headers['Access-Control-Allow-Origin'] = "*"
- response.headers['Access-Control-Allow-Methods'] = "POST, PUT, GET, DELETE"
- response.headers['Access-Control-Allow-Headers'] = "X-CKAN-API-KEY, Content-Type"
+ response.headers['Access-Control-Allow-Methods'] = "POST, PUT, GET, DELETE, OPTIONS"
+ response.headers['Access-Control-Allow-Headers'] = "X-CKAN-API-KEY, Authorization, Content-Type"
def _get_user(self, reference):
return model.User.by_name(reference)
diff --git a/ckan/lib/celery_app.py b/ckan/lib/celery_app.py
index cd87aa7..27d2951 100644
--- a/ckan/lib/celery_app.py
+++ b/ckan/lib/celery_app.py
@@ -1,5 +1,6 @@
import ConfigParser
import os
+from pylons import config as pylons_config
from pkg_resources import iter_entry_points
#from celery.loaders.base import BaseLoader
@@ -12,16 +13,22 @@
config = ConfigParser.ConfigParser()
config_file = os.environ.get('CKAN_CONFIG')
+
if not config_file:
config_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)), '../../development.ini')
config.read(config_file)
+sqlalchemy_url = pylons_config.get('sqlalchemy.url')
+if not sqlalchemy_url:
+ sqlalchemy_url = config.get('app:main', 'sqlalchemy.url')
+
+
default_config = dict(
BROKER_BACKEND = 'sqlalchemy',
- BROKER_HOST = config.get('app:main', 'sqlalchemy.url'),
- CELERY_RESULT_DBURI = config.get('app:main', 'sqlalchemy.url'),
+ BROKER_HOST = sqlalchemy_url,
+ CELERY_RESULT_DBURI = sqlalchemy_url,
CELERY_RESULT_BACKEND = 'database',
CELERY_RESULT_SERIALIZER = 'json',
CELERY_TASK_SERIALIZER = 'json',
diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py
index 18e81be..44d5c64 100644
--- a/ckan/lib/create_test_data.py
+++ b/ckan/lib/create_test_data.py
@@ -6,13 +6,16 @@ class CreateTestData(cli.CkanCommand):
'''Create test data in the database.
Tests can also delete the created objects easily with the delete() method.
- create-test-data - annakarenina and warandpeace
- create-test-data search - realistic data to test search
- create-test-data gov - government style data
- create-test-data family - package relationships data
- create-test-data user - create a user 'tester' with api key 'tester'
+ create-test-data - annakarenina and warandpeace
+ create-test-data search - realistic data to test search
+ create-test-data gov - government style data
+ create-test-data family - package relationships data
+ create-test-data user - create a user 'tester' with api key 'tester'
+ create-test-data translations - annakarenina, warandpeace, and some test
+ translations of terms
create-test-data vocabs - annakerenina, warandpeace, and some test
vocabularies
+
'''
summary = __doc__.split('\n')[0]
usage = __doc__
@@ -53,6 +56,8 @@ def command(self):
self.create_gov_test_data()
elif cmd == 'family':
self.create_family_test_data()
+ elif cmd == 'translations':
+ self.create_translations_test_data()
elif cmd == 'vocabs':
self.create_vocabs_test_data()
else:
@@ -92,6 +97,44 @@ def create_test_user(cls):
cls.user_refs.append(u'tester')
@classmethod
+
+ def create_translations_test_data(cls):
+ import ckan.model
+ CreateTestData.create()
+ rev = ckan.model.repo.new_revision()
+ rev.author = CreateTestData.author
+ rev.message = u'Creating test translations.'
+
+ sysadmin_user = ckan.model.User.get('testsysadmin')
+ package = ckan.model.Package.get('annakarenina')
+
+ # Add some new tags to the package.
+ # These tags are codes that are meant to be always translated before
+ # display, if not into the user's current language then into the
+ # fallback language.
+ package.add_tags([ckan.model.Tag('123'), ckan.model.Tag('456'),
+ ckan.model.Tag('789')])
+
+ # Add the above translations to CKAN.
+ for (lang_code, translations) in (('de', german_translations),
+ ('fr', french_translations), ('en', english_translations)):
+ for term in terms:
+ if term in translations:
+ data_dict = {
+ 'term': term,
+ 'term_translation': translations[term],
+ 'lang_code': lang_code,
+ }
+ context = {
+ 'model': ckan.model,
+ 'session': ckan.model.Session,
+ 'user': sysadmin_user.name,
+ }
+ ckan.logic.action.update.term_translation_update(context,
+ data_dict)
+
+ ckan.model.Session.commit()
+
def create_vocabs_test_data(cls):
import ckan.model
CreateTestData.create()
@@ -770,3 +813,49 @@ def get_all_data(cls):
}
}
]
+
+# Some test terms and translations.
+terms = ('A Novel By Tolstoy',
+ 'Index of the novel',
+ 'russian',
+ 'tolstoy',
+ "Dave's books",
+ "Roger's books",
+ 'Other (Open)',
+ 'romantic novel',
+ 'book',
+ '123',
+ '456',
+ '789',
+ 'plain text',
+ 'Roger likes these books.',
+)
+english_translations = {
+ '123': 'jealousy',
+ '456': 'realism',
+ '789': 'hypocrisy',
+}
+german_translations = {
+ 'A Novel By Tolstoy': 'Roman von Tolstoi',
+ 'Index of the novel': 'Index des Romans',
+ 'russian': 'Russisch',
+ 'tolstoy': 'Tolstoi',
+ "Dave's books": 'Daves Bucher',
+ "Roger's books": 'Rogers Bucher',
+ 'Other (Open)': 'Andere (Open)',
+ 'romantic novel': 'Liebesroman',
+ 'book': 'Buch',
+ '456': 'Realismus',
+ '789': 'Heuchelei',
+ 'plain text': 'Klartext',
+ 'Roger likes these books.': 'Roger mag diese Bucher.'
+}
+french_translations = {
+ 'A Novel By Tolstoy': 'A Novel par Tolstoi',
+ 'Index of the novel': 'Indice du roman',
+ 'russian': 'russe',
+ 'romantic novel': 'roman romantique',
+ 'book': 'livre',
+ '123': 'jalousie',
+ '789': 'hypocrisie',
+}
diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py
index d0e7d6c..e3bbd1c 100644
--- a/ckan/lib/dictization/model_dictize.py
+++ b/ckan/lib/dictization/model_dictize.py
@@ -1,7 +1,7 @@
import datetime
from pylons import config
from sqlalchemy.sql import select
-
+import datetime
import ckan.model
import ckan.misc
import ckan.logic as logic
@@ -32,6 +32,11 @@ def group_list_dictize(obj_list, context,
group_dict['packages'] = len(obj.active_packages().all())
+ if context.get('for_view'):
+ for item in plugins.PluginImplementations(
+ plugins.IGroupController):
+ group_dict = item.before_view(group_dict)
+
result_list.append(group_dict)
return sorted(result_list, key=sort_key, reverse=reverse)
@@ -153,6 +158,7 @@ def package_dictize(pkg, context):
q = q.where(resource_group.c.package_id == pkg.id)
result = _execute_with_revision(q, res_rev, context)
result_dict["resources"] = resource_list_dictize(result, context)
+
#tags
tag_rev = model.package_tag_revision_table
tag = model.tag_table
@@ -161,6 +167,14 @@ def package_dictize(pkg, context):
).where(tag_rev.c.package_id == pkg.id)
result = _execute_with_revision(q, tag_rev, context)
result_dict["tags"] = d.obj_list_dictize(result, context, lambda x: x["name"])
+
+ # Add display_names to tags. At first a tag's display_name is just the
+ # same as its name, but the display_name might get changed later (e.g.
+ # translated into another language by the multilingual extension).
+ for tag in result_dict['tags']:
+ assert not tag.has_key('display_name')
+ tag['display_name'] = tag['name']
+
#extras
extra_rev = model.extra_revision_table
q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id)
@@ -210,10 +224,9 @@ def package_dictize(pkg, context):
if pkg.metadata_created else None
if context.get('for_view'):
- for item in plugins.PluginImplementations(plugins.IPackageController):
+ for item in plugins.PluginImplementations( plugins.IPackageController):
result_dict = item.before_view(result_dict)
-
return result_dict
def _get_members(context, group, member_type):
@@ -226,7 +239,6 @@ def _get_members(context, group, member_type):
filter(model.Member.state == 'active').\
filter(model.Member.table_name == member_type[:-1]).all()
-
def group_dictize(group, context):
model = context['model']
result_dict = d.table_dictize(group, context)
@@ -279,6 +291,18 @@ def tag_dictize(tag, context):
result_dict = d.table_dictize(tag, context)
result_dict["packages"] = d.obj_list_dictize(tag.packages, context)
+
+ # Add display_names to tags. At first a tag's display_name is just the
+ # same as its name, but the display_name might get changed later (e.g.
+ # translated into another language by the multilingual extension).
+ assert not result_dict.has_key('display_name')
+ result_dict['display_name'] = result_dict['name']
+
+ if context.get('for_view'):
+ for item in plugins.PluginImplementations(
+ plugins.ITagController):
+ result_dict = item.before_view(result_dict)
+
return result_dict
def user_list_dictize(obj_list, context,
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index b4dd06e..dfdc0a8 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -7,6 +7,7 @@
"""
import email.utils
import datetime
+import logging
import re
import urllib
@@ -45,6 +46,8 @@
except ImportError:
import simplejson as json
+_log = logging.getLogger(__name__)
+
def redirect_to(*args, **kw):
'''A routes.redirect_to wrapper to retain the i18n settings'''
kw['__ckan_no_root'] = True
@@ -331,8 +334,41 @@ def _subnav_named_route(text, routename, **kwargs):
def default_group_type():
return str( config.get('ckan.default.group_type', 'group') )
+def unselected_facet_items(facet, limit=10):
+ '''Return the list of unselected facet items for the given facet, sorted
+ by count.
+
+ Returns the list of unselected facet contraints or facet items (e.g. tag
+ names like "russian" or "tolstoy") for the given search facet (e.g.
+ "tags"), sorted by facet item count (i.e. the number of search results that
+ match each facet item).
+
+ Reads the complete list of facet items for the given facet from
+ c.search_facets, and filters out the facet items that the user has already
+ selected.
+
+ Arguments:
+ facet -- the name of the facet to filter.
+ limit -- the max. number of facet items to return.
+
+ '''
+ if not c.search_facets or \
+ not c.search_facets.get(facet) or \
+ not c.search_facets.get(facet).get('items'):
+ return []
+ facets = []
+ for facet_item in c.search_facets.get(facet)['items']:
+ if not len(facet_item['name'].strip()):
+ continue
+ if not (facet, facet_item['name']) in request.params.items():
+ facets.append(facet_item)
+ return sorted(facets, key=lambda item: item['count'], reverse=True)[:limit]
def facet_items(*args, **kwargs):
+ """
+ DEPRECATED: Use the new facet data structure, and `unselected_facet_items()`
+ """
+ _log.warning('Deprecated function: ckan.lib.helpers:facet_items(). Will be removed in v1.8')
# facet_items() used to need c passing as the first arg
# this is depriciated as pointless
# throws error if ckan.restrict_template_vars is True
@@ -707,3 +743,67 @@ def process_names(items):
item = converter(obj, context)
items.append(item)
return items
+
+
+# these are the functions that will end up in `h` template helpers
+# if config option restrict_template_vars is true
+__allowed_functions__ = [
+ # functions defined in ckan.lib.helpers
+ 'redirect_to',
+ 'url',
+ 'url_for',
+ 'url_for_static',
+ 'lang',
+ 'flash',
+ 'flash_error',
+ 'flash_notice',
+ 'flash_success',
+ 'nav_link',
+ 'nav_named_link',
+ 'subnav_link',
+ 'subnav_named_route',
+ 'default_group_type',
+ 'facet_items',
+ 'facet_title',
+ # am_authorized, # depreciated
+ 'check_access',
+ 'linked_user',
+ 'linked_authorization_group',
+ 'group_name_to_title',
+ 'markdown_extract',
+ 'icon',
+ 'icon_html',
+ 'icon_url',
+ 'resource_icon',
+ 'format_icon',
+ 'linked_gravatar',
+ 'gravatar',
+ 'pager_url',
+ 'render_datetime',
+ 'date_str_to_datetime',
+ 'datetime_to_date_str',
+ 'parse_rfc_2822_date',
+ 'time_ago_in_words_from_str',
+ 'button_attr',
+ 'dataset_display_name',
+ 'dataset_link',
+ 'resource_display_name',
+ 'resource_link',
+ 'tag_link',
+ 'group_link',
+ 'dump_json',
+ 'auto_log_message',
+ 'snippet',
+ 'convert_to_dict',
+ 'activity_div',
+ # imported into ckan.lib.helpers
+ 'literal',
+ 'link_to',
+ 'get_available_locales',
+ 'get_locales_dict',
+ 'truncate',
+ 'file',
+ 'mail_to',
+ 'radio',
+ 'submit',
+]
diff --git a/ckan/lib/helpers_clean.py b/ckan/lib/helpers_clean.py
deleted file mode 100644
index 54c50c8..0000000
--- a/ckan/lib/helpers_clean.py
+++ /dev/null
@@ -1,119 +0,0 @@
-''' This file is part of a plan to clean up the ckan.lib.helpers whilst
-keeping existing versions of ckan from breaking.
-
-
-When it is decided that we will make the template var cleanup
-permanent we will need to implement this using __all__ = [...] in
-lib.helpers itself. Unused imports can also be removed at this time.
-
-yes, yes `from ... import ...` is the work of Satan but this is just a
-short term botch :)
-
-'''
-
-
-from ckan.lib.helpers import (
- # functions defined in ckan.lib.helpers
- redirect_to,
- url,
- url_for,
- url_for_static,
- lang,
- flash,
- flash_error,
- flash_notice,
- flash_success,
- nav_link,
- nav_named_link,
- subnav_link,
- subnav_named_route,
- default_group_type,
- facet_items,
- facet_title,
- # am_authorized, # depreciated
- check_access,
- linked_user,
- linked_authorization_group,
- group_name_to_title,
- markdown_extract,
- icon,
- icon_html,
- icon_url,
- resource_icon,
- format_icon,
- linked_gravatar,
- gravatar,
- pager_url,
- render_datetime,
- date_str_to_datetime,
- datetime_to_date_str,
- parse_rfc_2822_date,
- time_ago_in_words_from_str,
- button_attr,
- dataset_display_name,
- dataset_link,
- resource_display_name,
- resource_link,
- tag_link,
- group_link,
- dump_json,
- auto_log_message,
- snippet,
- convert_to_dict,
- activity_div,
- # imported into ckan.lib.helpers
- literal,
- link_to,
- get_available_locales,
- get_locales_dict,
- truncate,
- file,
- mail_to,
- radio,
- submit,
-)
-
-
-# these are potentially used by templates but hopefully are not
-imported_functions = [
- 'are_there_flash_messages',
- 'auto_discovery_link',
- # 'beaker_cache',
- 'checkbox',
- # 'ckan',
- # 'config',
- 'convert_boolean_attrs',
- 'css_classes',
- 'date',
- 'datetime',
- 'email',
- 'end_form',
- 'escape',
- 'form',
- 'fromstring',
- 'hidden',
- 'i18n',
- 'image',
- 'javascript_link',
- 'json',
- 'link_to_if',
- 'link_to_unless',
- 'markdown',
- 'ol',
- 'paginate',
- 'password',
- # 're',
- 'request',
- 'required_legend',
- 'select',
- 'stylesheet_link',
- 'text',
- 'textarea',
- 'th_sortable',
- 'title',
- 'ul',
- 'url_escape',
- 'urllib',
- 'xml_declaration',
-]
-
diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py
index 513f291..30cd4ae 100644
--- a/ckan/logic/__init__.py
+++ b/ckan/logic/__init__.py
@@ -39,6 +39,7 @@ class NotAuthorized(ActionError):
class ParameterError(ActionError):
pass
+
class ValidationError(ParameterError):
def __init__(self, error_dict, error_summary=None, extra_msg=None):
self.error_dict = error_dict
@@ -224,3 +225,29 @@ def get_action(action):
_actions.update(fetched_actions)
return _actions.get(action)
+def get_or_bust(data_dict, keys):
+ '''Try and get values from dictionary and if they are not there
+ raise a validataion error.
+
+ data_dict: a dictionary
+ keys: either a single string key in which case will return a single value,
+ or a iterable which will return a tuple for unpacking purposes.
+
+ e.g single_value = get_or_bust(data_dict, 'a_key')
+ value_1, value_2 = get_or_bust(data_dict, ['key1', 'key2'])
+ '''
+ values = []
+ errors = {}
+
+ if isinstance(keys, basestring):
+ keys = [keys]
+ for key in keys:
+ value = data_dict.get(key)
+ if not value:
+ errors[key] = _('Missing value')
+ values.append(value)
+ if errors:
+ raise ValidationError(errors)
+ if len(values) == 1:
+ return values[0]
+ return tuple(values)
diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
index 21429d3..b50d53c 100644
--- a/ckan/logic/action/get.py
+++ b/ckan/logic/action/get.py
@@ -6,7 +6,7 @@
import webhelpers.html
from sqlalchemy.sql import select
from sqlalchemy.orm import aliased
-from sqlalchemy import or_, and_, func, desc, case
+from sqlalchemy import or_, and_, func, desc, case, text
import ckan
import ckan.authz
@@ -31,6 +31,7 @@
check_access = logic.check_access
NotFound = logic.NotFound
ValidationError = logic.ValidationError
+get_or_bust = logic.get_or_bust
def _package_list_with_resources(context, package_revision_list):
package_list = []
@@ -417,9 +418,27 @@ def resource_show(context, data_dict):
raise NotFound
check_access('resource_show', context, data_dict)
-
return model_dictize.resource_dictize(resource, context)
+def resource_status_show(context, data_dict):
+
+ model = context['model']
+ id = get_or_bust(data_dict, 'id')
+
+ check_access('resource_status_show', context, data_dict)
+
+ # needs to be text query as celery tables are not in our model
+ q = text("""select status, date_done, traceback, task_status.*
+ from task_status left join celery_taskmeta
+ on task_status.value = celery_taskmeta.task_id and key = 'celery_task_id'
+ where entity_id = :entity_id """)
+
+ result = model.Session.connection().execute(q, entity_id=id)
+ result_list = [table_dictize(row, context) for row in result]
+
+ return result_list
+
+
def revision_show(context, data_dict):
model = context['model']
api = context.get('api_version')
@@ -753,12 +772,40 @@ def package_search(context, data_dict):
'results': results
}
+ # Transform facets into a more useful data structure.
+ restructured_facets = {}
+ for key, value in search_results['facets'].items():
+ restructured_facets[key] = {
+ 'title': key,
+ 'items': []
+ }
+ for key_, value_ in value.items():
+ new_facet_dict = {}
+ new_facet_dict['name'] = key_
+ if key == 'groups':
+ group = model.Group.get(key_)
+ if group:
+ new_facet_dict['display_name'] = group.display_name
+ else:
+ new_facet_dict['display_name'] = key_
+ else:
+ new_facet_dict['display_name'] = key_
+ new_facet_dict['count'] = value_
+ restructured_facets[key]['items'].append(new_facet_dict)
+ search_results['search_facets'] = restructured_facets
+
# check if some extension needs to modify the search results
for item in plugins.PluginImplementations(plugins.IPackageController):
search_results = item.after_search(search_results,data_dict)
- return search_results
+ # After extensions have had a chance to modify the facets, sort them by
+ # display name.
+ for facet in search_results['search_facets']:
+ search_results['search_facets'][facet]['items'] = sorted(
+ search_results['search_facets'][facet]['items'],
+ key=lambda facet: facet['display_name'], reverse=True)
+ return search_results
def resource_search(context, data_dict):
model = context['model']
@@ -933,13 +980,13 @@ def term_translation_show(context, data_dict):
q = select([trans_table])
- if 'term' not in data_dict:
- raise ValidationError({'term': 'term not in data'})
+ if 'terms' not in data_dict:
+ raise ValidationError({'terms': 'terms not in data'})
- q = q.where(trans_table.c.term == data_dict['term'])
+ q = q.where(trans_table.c.term.in_(data_dict['terms']))
- if 'lang_code' in data_dict:
- q = q.where(trans_table.c.lang_code == data_dict['lang_code'])
+ if 'lang_codes' in data_dict:
+ q = q.where(trans_table.c.lang_code.in_(data_dict['lang_codes']))
conn = model.Session.connection()
cursor = conn.execute(q)
diff --git a/ckan/logic/auth/get.py b/ckan/logic/auth/get.py
index 27e8436..9a04520 100644
--- a/ckan/logic/auth/get.py
+++ b/ckan/logic/auth/get.py
@@ -154,6 +154,9 @@ def format_autocomplete(context, data_dict):
def task_status_show(context, data_dict):
return {'success': True}
+def resource_status_show(context, data_dict):
+ return {'success': True}
+
## Modifications for rest api
def package_show_rest(context, data_dict):
diff --git a/ckan/logic/auth/publisher/get.py b/ckan/logic/auth/publisher/get.py
index b2dbd9a..f640625 100644
--- a/ckan/logic/auth/publisher/get.py
+++ b/ckan/logic/auth/publisher/get.py
@@ -160,6 +160,9 @@ def format_autocomplete(context, data_dict):
def task_status_show(context, data_dict):
return {'success': True}
+def resource_status_show(context, data_dict):
+ return {'success': True}
+
## Modifications for rest api
def package_show_rest(context, data_dict):
diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py
index b07d2bb..0d41722 100644
--- a/ckan/logic/schema.py
+++ b/ckan/logic/schema.py
@@ -85,6 +85,7 @@ def default_tags_schema():
'vocabulary_id': [ignore_missing, unicode, vocabulary_id_exists],
'revision_timestamp': [ignore],
'state': [ignore],
+ 'display_name': [ignore],
}
return schema
diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py
index df0d2f5..f254318 100644
--- a/ckan/model/__init__.py
+++ b/ckan/model/__init__.py
@@ -79,6 +79,16 @@ def init_db(self):
else:
if not self.tables_created_and_initialised:
self.upgrade_db()
+ ## make sure celery tables are made as celery only makes them after
+ ## adding a task
+ try:
+ import ckan.lib.celery_app as celery_app
+ backend = celery_app.celery.backend
+ ##This creates the database tables as a side effect, can not see another way
+ ##to make tables unless you actually create a task.
+ celery_result_session = backend.ResultSession()
+ except ImportError:
+ pass
self.init_configuration_data()
self.tables_created_and_initialised = True
diff --git a/ckan/plugins/__init__.py b/ckan/plugins/__init__.py
index e11e909..719a24a 100644
--- a/ckan/plugins/__init__.py
+++ b/ckan/plugins/__init__.py
@@ -1,19 +1,7 @@
from ckan.plugins.core import *
from ckan.plugins.interfaces import *
-
-class _Toolkit(object):
- ''' This object allows us to avoid circular imports while making
- functions/objects available to plugins. '''
-
- def __init__(self):
- self.toolkit = None
-
- def __getattr__(self, name):
- if not self.toolkit:
- import toolkit
- self.toolkit = toolkit
- return getattr(self.toolkit, name)
-
-toolkit = _Toolkit()
-del _Toolkit
+# Expose the toolkit object without doing an import *
+import toolkit as _toolkit
+toolkit = _toolkit.toolkit
+del _toolkit
diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py
index ee092d7..bc2eb88 100644
--- a/ckan/plugins/interfaces.py
+++ b/ckan/plugins/interfaces.py
@@ -13,7 +13,9 @@
'IPackageController', 'IPluginObserver',
'IConfigurable', 'IConfigurer', 'IAuthorizer',
'IActions', 'IResourceUrlChange', 'IDatasetForm',
- 'IGroupForm', 'ITemplateHelpers',
+ 'IGroupForm',
+ 'ITagController',
+ 'ITemplateHelpers',
]
from inspect import isclass
@@ -181,6 +183,21 @@ class IResourceUrlChange(Interface):
def notify(self, resource):
pass
+class ITagController(Interface):
+ '''
+ Hook into the Tag controller. These will usually be called just before
+ committing or returning the respective object, i.e. all validation,
+ synchronization and authorization setup are complete.
+
+ '''
+ def before_view(self, tag_dict):
+ '''
+ Extensions will recieve this before the tag gets displayed. The
+ dictionary passed will be the one that gets sent to the template.
+
+ '''
+ return tag_dict
+
class IGroupController(Interface):
"""
Hook into the Group controller. These will
diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py
index e635c43..3eb8983 100644
--- a/ckan/plugins/toolkit.py
+++ b/ckan/plugins/toolkit.py
@@ -1,10 +1,3 @@
-## This file is intended to make functions/objects consistently
-## available to plugins whilst giving developers the ability move code
-## around or change underlying frameworks etc. It should not be used
-## internally within ckan only by extensions. Functions should only be
-## removed from this file after reasonable depreciation notice has
-## been given.
-
import inspect
import os
import re
@@ -13,122 +6,184 @@
import paste.deploy.converters as converters
import webhelpers.html.tags
-import ckan
-import ckan.lib.base as base
-import ckan.logic as logic
-import ckan.lib.cli as cli
-
-
-
-__all__ = [
- ## Imported functions/objects ##
- '_', # i18n translation
- 'c', # template context
- 'request', # http request object
- 'render', # template render function
- 'render_text', # Genshi NewTextTemplate render function
- 'render_snippet', # snippet render function
- 'asbool', # converts an object to a boolean
- 'asint', # converts an object to an integer
- 'aslist', # converts an object to a list
- 'literal', # stop tags in a string being escaped
- 'get_action', # get logic action function
- 'check_access', # check logic function authorisation
- 'ActionNotFound', # action not found exception (ckan.logic.NotFound)
- 'NotAuthorized', # action not authorized exception
- 'ValidationError', # model update validation error
- 'CkanCommand', # class for providing cli interfaces
-
- ## Fully defined in this file ##
- 'add_template_directory',
- 'add_public_directory',
- 'requires_ckan_version',
- 'check_ckan_version',
- 'CkanVersionException',
-]
-
-_ = pylons.i18n._
-c = pylons.c
-request = pylons.request
-render = base.render
-render_text = base.render_text
-asbool = converters.asbool
-asint = converters.asint
-aslist = converters.aslist
-literal = webhelpers.html.tags.literal
-
-get_action = logic.get_action
-check_access = logic.check_access
-ActionNotFound = logic.NotFound ## Name change intentional
-NotAuthorized = logic.NotAuthorized
-ValidationError = logic.ValidationError
-
-CkanCommand = cli.CkanCommand
-
-# wrappers
-def render_snippet(template, data=None):
- data = data or {}
- return base.render_snippet(template, **data)
-
-
-# new functions
-def add_template_directory(config, relative_path):
- ''' Function to aid adding extra template paths to the config.
- The path is relative to the file calling this function. '''
- _add_served_directory(config, relative_path, 'extra_template_paths')
-
-def add_public_directory(config, relative_path):
- ''' Function to aid adding extra public paths to the config.
- The path is relative to the file calling this function. '''
- _add_served_directory(config, relative_path, 'extra_public_paths')
-
-def _add_served_directory(config, relative_path, config_var):
- ''' Add extra public/template directories to config. '''
- assert config_var in ('extra_template_paths', 'extra_public_paths')
- # we want the filename that of the function caller but they will
- # have used one of the available helper functions
- frame, filename, line_number, function_name, lines, index =\
- inspect.getouterframes(inspect.currentframe())[2]
-
- this_dir = os.path.dirname(filename)
- absolute_path = os.path.join(this_dir, relative_path)
- if absolute_path not in config.get(config_var, ''):
- if config.get(config_var):
- config[config_var] += ',' + absolute_path
- else:
- config[config_var] = absolute_path
+__all__ = ['toolkit']
class CkanVersionException(Exception):
''' Exception raised if required ckan version is not available. '''
pass
-
-def _version_str_2_list(v_str):
- ''' conver a version string into a list of ints
- eg 1.6.1b --> [1, 6, 1] '''
- v_str = re.sub(r'[^0-9.]', '', v_str)
- return [int(part) for part in v_str.split('.')]
-
-def check_ckan_version(min_version=None, max_version=None):
- ''' Check that the ckan version is correct for the plugin. '''
- current = _version_str_2_list(ckan.__version__)
-
- if min_version:
- min_required = _version_str_2_list(min_version)
- if current < min_required:
- return False
- if max_version:
- max_required = _version_str_2_list(max_version)
- if current > max_required:
- return False
- return True
-
-def requires_ckan_version(min_version, max_version=None):
- ''' Check that the ckan version is correct for the plugin. '''
- if not check_ckan_version(min_version=min_version, max_version=max_version):
- if not max_version:
- error = 'Requires ckan version %s or higher' % min_version
+class _Toolkit(object):
+ '''This class is intended to make functions/objects consistently
+ available to plugins, whilst giving developers the ability move
+ code around or change underlying frameworks etc. This object allows
+ us to avoid circular imports while making functions/objects
+ available to plugins.
+
+ It should not be used internally within ckan only by extensions.
+
+ Functions/objects should only be removed after reasonable
+ depreciation notice has been given.'''
+
+ # contents should describe the available functions/objects. We check
+ # that this list matches the actual availables in the initialisation
+ contents = [
+ ## Imported functions/objects ##
+ '_', # i18n translation
+ 'c', # template context
+ 'request', # http request object
+ 'render', # template render function
+ 'render_text', # Genshi NewTextTemplate render function
+ 'render_snippet', # snippet render function
+ 'asbool', # converts an object to a boolean
+ 'asint', # converts an object to an integer
+ 'aslist', # converts an object to a list
+ 'literal', # stop tags in a string being escaped
+ 'get_action', # get logic action function
+ 'check_access', # check logic function authorisation
+ 'ActionNotFound', # action not found exception (ckan.logic.NotFound)
+ 'NotAuthorized', # action not authorized exception
+ 'ValidationError', # model update validation error
+ 'CkanCommand', # class for providing cli interfaces
+
+ ## Fully defined in this file ##
+ 'add_template_directory',
+ 'add_public_directory',
+ 'requires_ckan_version',
+ 'check_ckan_version',
+ 'CkanVersionException',
+ ]
+
+
+ def __init__(self):
+ self._toolkit = {}
+
+ def _initialize(self):
+ ''' get the required functions/objects, store them for later
+ access and check that they match the contents dict. '''
+
+ import ckan
+ import ckan.lib.base as base
+ import ckan.logic as logic
+ import ckan.lib.cli as cli
+
+ # Allow class access to these modules
+ self.__class__.ckan = ckan
+ self.__class__.base = base
+
+ t = self._toolkit
+
+ # imported functions
+ t['_'] = pylons.i18n._
+ t['c'] = pylons.c
+ t['request'] = pylons.request
+ t['render'] = base.render
+ t['render_text'] = base.render_text
+ t['asbool'] = converters.asbool
+ t['asint'] = converters.asint
+ t['aslist'] = converters.aslist
+ t['literal'] = webhelpers.html.tags.literal
+
+ t['get_action'] = logic.get_action
+ t['check_access'] = logic.check_access
+ t['ActionNotFound'] = logic.NotFound ## Name change intentional
+ t['NotAuthorized'] = logic.NotAuthorized
+ t['ValidationError'] = logic.ValidationError
+
+ t['CkanCommand'] = cli.CkanCommand
+
+ # class functions
+ t['render_snippet'] = self._render_snippet
+ t['add_template_directory'] = self._add_template_directory
+ t['add_public_directory'] = self._add_public_directory
+ t['requires_ckan_version'] = self._requires_ckan_version
+ t['check_ckan_version'] = self._check_ckan_version
+ t['CkanVersionException'] = CkanVersionException
+
+ # check contents list correct
+ errors = set(t).symmetric_difference(set(self.contents))
+ if errors:
+ raise Exception('Plugin toolkit error %s not matching' % errors)
+
+ # wrappers
+ @classmethod
+ def _render_snippet(cls, template, data=None):
+ ''' helper for the render_snippet function as it uses keywords
+ rather than dict to pass data '''
+ data = data or {}
+ return cls.base.render_snippet(template, **data)
+
+ # new functions
+ @classmethod
+ def _add_template_directory(cls, config, relative_path):
+ ''' Function to aid adding extra template paths to the config.
+ The path is relative to the file calling this function. '''
+ cls._add_served_directory(config, relative_path, 'extra_template_paths')
+
+ @classmethod
+ def _add_public_directory(cls, config, relative_path):
+ ''' Function to aid adding extra public paths to the config.
+ The path is relative to the file calling this function. '''
+ cls._add_served_directory(config, relative_path, 'extra_public_paths')
+
+ @classmethod
+ def _add_served_directory(cls, config, relative_path, config_var):
+ ''' Add extra public/template directories to config. '''
+ assert config_var in ('extra_template_paths', 'extra_public_paths')
+ # we want the filename that of the function caller but they will
+ # have used one of the available helper functions
+ frame, filename, line_number, function_name, lines, index =\
+ inspect.getouterframes(inspect.currentframe())[2]
+
+ this_dir = os.path.dirname(filename)
+ absolute_path = os.path.join(this_dir, relative_path)
+ if absolute_path not in config.get(config_var, ''):
+ if config.get(config_var):
+ config[config_var] += ',' + absolute_path
+ else:
+ config[config_var] = absolute_path
+
+ @classmethod
+ def _version_str_2_list(cls, v_str):
+ ''' convert a version string into a list of ints
+ eg 1.6.1b --> [1, 6, 1] '''
+ v_str = re.sub(r'[^0-9.]', '', v_str)
+ return [int(part) for part in v_str.split('.')]
+
+ @classmethod
+ def _check_ckan_version(cls, min_version=None, max_version=None):
+ ''' Check that the ckan version is correct for the plugin. '''
+ current = cls._version_str_2_list(cls.ckan.__version__)
+
+ if min_version:
+ min_required = cls._version_str_2_list(min_version)
+ if current < min_required:
+ return False
+ if max_version:
+ max_required = cls._version_str_2_list(max_version)
+ if current > max_required:
+ return False
+ return True
+
+ @classmethod
+ def _requires_ckan_version(cls, min_version, max_version=None):
+ ''' Check that the ckan version is correct for the plugin. '''
+ if not cls._check_ckan_version(min_version=min_version,
+ max_version=max_version):
+ if not max_version:
+ error = 'Requires ckan version %s or higher' % min_version
+ else:
+ error = 'Requires ckan version between %s and %s' % \
+ (min_version, max_version)
+ raise cls.CkanVersionException(error)
+
+ def __getattr__(self, name):
+ ''' return the function/object requested '''
+ if not self._toolkit:
+ self._initialize()
+ if name in self._toolkit:
+ return self._toolkit[name]
else:
- error = 'Requires ckan version between %s and %s' % \
- (min_version, max_version)
- raise CkanVersionException(error)
+ raise Exception('`%s` not found in plugins toolkit' % name)
+
+toolkit = _Toolkit()
+del _Toolkit
diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js
index 995e703..d94bbd4 100644
--- a/ckan/public/scripts/application.js
+++ b/ckan/public/scripts/application.js
@@ -958,7 +958,6 @@ CKAN.Utils = function($, my) {
, select: function(event, ui) {
var input_box = $(this);
input_box.val('');
- var parent_dd = input_box.parent('dd');
var old_name = input_box.attr('name');
var field_name_regex = /^(\S+)__(\d+)__(\S+)$/;
var split = old_name.match(field_name_regex);
@@ -967,10 +966,15 @@ CKAN.Utils = function($, my) {
input_box.attr('name', new_name);
input_box.attr('id', new_name);
-
- parent_dd.before(
- '<input type="hidden" name="' + old_name + '" value="' + ui.item.value + '">' + '<dd>' + ui.item.label + '</dd>'
- );
+
+ var $new = $('<div class="ckan-dataset-to-add"><p></p></div>');
+ $new.append($('<input type="hidden" />').attr('name', old_name).val(ui.item.value));
+ $new.append('<i class="icon-plus-sign"></i> ');
+ $new.append(ui.item.label);
+ input_box.after($new);
+
+ // prevent setting value in autocomplete box
+ return false;
}
});
};
diff --git a/ckan/templates/_snippet/data-api-help.html b/ckan/templates/_snippet/data-api-help.html
index 5a499c9..bab65b2 100644
--- a/ckan/templates/_snippet/data-api-help.html
+++ b/ckan/templates/_snippet/data-api-help.html
@@ -15,102 +15,51 @@
</h3>
</div>
<div class="modal-body">
- <p><strong>Access resource data via a web API with powerful query
- support</strong>. Further information in the <a
- href="http://docs.ckan.org/en/latest/storage/datastore.html" target="_blank">main
- CKAN Data API and DataStore documentation</a>.</p>
+ <p><strong>Access (and update) resource data via a web API with powerful query
+ support</strong>.</p>
+
+ <ul>
+ <li>Quickstart info below</li>
+ <li><a
+ href="http://docs.ckan.org/en/latest/using-data-api.html" target="_blank">
+ Tutorial, examples and full details in main CKAN documentation</a></li>
+ </ul>
<div class="accordion-group">
<div class="accordion-heading">
- <a class="accordion-toggle" data-toggle="collapse" href=".collapse-endpoints">Endpoints »</a>
- </div>
- <div class="collapse-endpoints in accordion-body collapse">
- <div class="accordion-inner">
- <p>The Data API builds directly on ElasticSearch, with a resource API
- endpoint being equivalent to a single index 'type' in ElasticSearch.
- This means you can directly re-use <a
- href="http://www.elasticsearch.org/guide/appendix/clients.html"
- _target="blank">ElasticSearch
- client libraries</a> when connecting to the API endpoint.</p>
- <table class="table-condensed table-striped table-bordered">
- <thead></thead>
- <tbody>
- <tr>
- <th>Query example</th>
- <td>
- <code><a href="${datastore_api}/_search?size=5&pretty=true" target="_blank">${datastore_api}/_search?size=5&pretty=true</a></code>
- </td>
- </tr>
- <tr>
- <th>Schema (Mapping)</th>
- <td>
- <code><a href="${datastore_api}/_mapping" target="_blank">${datastore_api}/_mapping?pretty=true</a></code>
- </td>
- </tr>
- <tr>
- <th>Base</th>
- <td><code>${datastore_api}</code></td>
- </tr>
- </tbody>
- </table>
- </div>
- </div>
- </div>
-
- <div class="accordion-group">
- <div class="accordion-heading">
- <a class="accordion-toggle" href=".collapse-querying" data-toggle="collapse">Querying
- »</a>
+ <a class="accordion-toggle" data-toggle="collapse" href=".collapse-endpoints">Example URLs and Endpoints»</a>
</div>
- <div class="collapse collapse-querying accordion-body in">
+ <div class="collapse-endpoints in accordion-body">
<div class="accordion-inner">
- <p><strong>Basic queries</strong> can be done using the <code>q</code>
- parameter in the query string which supports the <a
- href="http://lucene.apache.org/core/old_versioned_docs/versions/3_0_0/queryparsersyntax.html"
- target="_blank">Lucene
- query parser syntax</a> and hence filters on specific fields
- (<code>e.g. fieldname:value</code>), wildcards (e.g. <code>abc*</code>)
- and more. Full query parameters and options in the <a
- href="http://www.elasticsearch.org/guide/reference/api/search/uri-request.html"
- target="_blank">ElasticSearch
- URI request docs</a>.</p>
+ <strong>Query example (first 5 results)</strong>
+ <p>
+ <code><a href="${datastore_api}/_search?size=5&pretty=true" target="_blank">${datastore_api}/_search?size=5&pretty=true</a></code>
+ </p>
+
+ <strong>Query example (results with 'jones' in <code>title</code> field)</strong>
+ <p>
+ <code><a href="${datastore_api}/_search?q=title:jones&size=5&pretty=true"
+ target="_blank">${datastore_api}/_search?q=title:jones&size=5&pretty=true</a></code>
+ </p>
- <p><strong>More complex queries</strong>, including those that involve
- faceting and statistical operations, should use the full ElasticSearch
- query language in which the query is a JSON structure sent in the
- <code>?source=</code> parameter. See <a
- href="http://www.elasticsearch.org/guide/reference/api/search/"
- target="_blank">ElasticSearch
- query documentation</a>.</p>
+ <strong>Schema (Mapping)</strong>
+ <p>
+ <code><a href="${datastore_api}/_mapping" target="_blank">${datastore_api}/_mapping?pretty=true</a></code>
+ </p>
- <p>JSONP support is available via a simple callback query parameter:
- <code>?callback=my_callback_name</code>.</p>
+ <strong>Endpoint (for clients)</strong>
+ <p>
+ <code>${datastore_api}</code>
+ </p>
</div>
</div>
</div>
<div class="accordion-group">
<div class="accordion-heading">
- <a class="accordion-toggle" href=".collapse-curl"
- data-toggle="collapse">Example: cURL (or Browser) »</a>
- </div>
- <div class="accordion-body collapse-curl collapse in">
- <div class="accordion-inner">
- <p>The following examples utilize the <a href="http://curl.haxx.se/">cURL</a>
- command line utility. If you prefer, you you can just open the relevant urls in
- your browser.</p>
-
- <pre>
-// added pretty=true to get the json results pretty printed
-curl ${datastore_api}/_search?q=title:jones&size=5&pretty=true</pre>
- </div>
- </div>
- </div>
- <div class="accordion-group">
- <div class="accordion-heading">
<a class="accordion-toggle" href=".collapse-javascript" data-toggle="collapse">Example: Javascript</a>
</div>
- <div class="accordion-body collapse-javascript collapse in">
+ <div class="accordion-body collapse-javascript in">
<div class="accordion-inner">
<p>A simple ajax (JSONP) request to the data API using jQuery.</p>
<pre>
@@ -119,7 +68,8 @@
q: 'title:jones' // query on the title field for 'jones'
};
$.ajax({
- url: ${datastore_api}/_search,
+ url: '${datastore_api}/_search',
+ data: data,
dataType: 'jsonp',
success: function(data) {
alert('Total results found: ' + data.hits.total)
@@ -128,6 +78,22 @@
</div>
</div>
</div>
+
+ <div class="accordion-group">
+ <div class="accordion-heading">
+ <a class="accordion-toggle" href=".collapse-javascript" data-toggle="collapse">Example: Python</a>
+ </div>
+ <div class="accordion-body collapse-javascript in">
+ <div class="accordion-inner">
+ <pre>
+import urllib
+url = '${datastore_api}/_search?size=5&q=title:jones'
+fileobj = urllib.urlopen(url)
+print fileobj.read()
+</pre>
+ </div>
+ </div>
+ </div>
</div>
</div>
diff --git a/ckan/templates/_util.html b/ckan/templates/_util.html
index f470b07..a98504f 100644
--- a/ckan/templates/_util.html
+++ b/ckan/templates/_util.html
@@ -40,19 +40,11 @@
<ul py:def="tag_list(tags)" class="tags clearfix">
<py:for each="tag in tags">
<li py:if="not tag.get('vocabulary_id')">
- ${h.link_to(tag['name'], h.url_for(controller='tag', action='read', id=tag['name']))}
+ ${h.link_to(tag['display_name'], h.url_for(controller='tag', action='read', id=tag['name']))}
</li>
</py:for>
</ul>
- <!--! List of tags: pass in a list of tag name and this renders the standard
- tag listing -->
- <ul py:def="tag_list_from_name(tags)" class="tags clearfix">
- <li py:for="tag in tags">
- ${h.link_to(tag, h.url_for(controller='tag', action='read', id=tag))}
- </li>
- </ul>
-
<!--! List of users: pass in a collection of users and this renders the standard
user listing -->
<ul py:def="user_list(users)" class="users">
diff --git a/ckan/templates/facets.html b/ckan/templates/facets.html
index f50ac3b..f8c104b 100644
--- a/ckan/templates/facets.html
+++ b/ckan/templates/facets.html
@@ -5,43 +5,176 @@
py:strip=""
>
+<!--
+Construct a facet <div> populated with links to filtered results.
+
+name
+ The field name identifying the facet field, eg. "tags"
+
+title
+ The title of the facet, eg. "Tags", or "Tag Cloud"
+
+label_function
+ Renders the human-readable label for each facet value.
+ If defined, this should be a callable that accepts a `facet_item`.
+ eg. lambda facet_item: facet_item.display_name.upper()
+ By default it displays the facet item's display name, which should
+ usually be good enough
+
+if_empty
+ A string, which if defined, and the list of possible facet items is empty,
+ is displayed in lieu of an empty list.
+
+count_label
+ A callable which accepts an integer, and returns a string. This controls
+ how a facet-item's count is displayed.
+
+-->
+<py:def function="facet_div(name, title, limit=10, label_function=lambda item: item.display_name, if_empty=None, count_label=lambda c: ' (%d)'%c)">
+ <div py:if="if_empty is not None or h.unselected_facet_items(name, limit)" class="facet-box">
+ <h2>${h.facet_title(title)}</h2>
+ <ul class="facet-options">
+ ${facet_li(name, limit=limit, label_function=label_function, if_empty=None, count_label=count_label)}
+ </ul>
+ <p py:if="not h.unselected_facet_items(name, limit)">${if_empty}</p>
+ </div>
+</py:def>
+
+<!--
+Generate <li>s for facet items. The generated tags are not wrapped by any
+other tag, ie - it's up to the caller to wrap them in something suitable.
+
+name
+ The field name identifying the facet field, eg. "tags"
+
+label_function
+ Renders the human-readable label for each facet value.
+ If defined, this should be a callable that accepts a `facet_item`.
+ eg. lambda facet_item: facet_item.display_name.upper()
+ By default it displays the facet item's display name, which should
+ usually be good enough
+
+if_empty
+ A string, which if defined, and the list of possible facet items is empty,
+ is displayed in a <li> tag in lieu of an empty list.
+
+count_label
+ A callable which accepts an integer, and returns a string. This controls
+ how a facet-item's count is displayed.
+
+-->
+<py:def function="facet_li(name, limit=5, label_function=lambda item: item.display_name, if_empty=None, count_label=lambda c: ' (%d)'%c)">
+
+ <li py:if="if_empty and not h.unselected_facet_items(name, limit)">${if_empty}</li>
+
+ <li py:for="facet_item in h.unselected_facet_items(name, limit)">
+ <a href="${c.drill_down_url(**{name: facet_item.name})}">
+ ${label_function(facet_item)}
+ </a>
+ ${count_label(facet_item['count'])}
+ </li>
+</py:def>
+
+<!--
+DEPRECATED. Provided only for backward compatibility with existing plugins.
+ Use `facet_div` instead.
+
+Similar to the above, `facet_div` function; this helper creates a <div>
+populated with links to filtered search results.
+
+Differences with the preferred `facet_div` function:
+
+ * `title` argument is a callable
+ * `label` is a callable that accepts a string cf. `label_function` which is
+ a callable that accepts a `facet_item`.
+
+code
+ The field name identifying the facet field, eg. "tags"
+
+title
+ A callable used to render the title for the facet. The callable must accept
+ one string argument, the `code` argument passed in above. Obviously, this
+ is a bit redundant.
+
+label
+ Renders the human-readable label for each facet value.
+ If defined, this should be a callable that accepts a facet_item's name as a
+ string.
+
+if_empty
+ A string, which if defined, and the list of possible facet items is empty,
+ is displayed in lieu of an empty list.
+
+count_label
+ A callable which accepts an integer, and returns a string. This controls
+ how a facet-item's count is displayed.
+
+-->
<py:def function="facet_sidebar(code, limit=5, label=lambda n: n, title=h.facet_title, if_empty=None, count_label=lambda c: ' (%d)'%c)">
- <div py:if="if_empty is not None or len(h.facet_items(code, limit=limit))" class="facet-box">
+ <?python
+ import logging
+ logging.getLogger('ckan.templates.facets').warning("Deprecated function: ckan/templates/facets.html:facet_sidebar()")
+ ?>
+ <div py:if="if_empty is not None or h.unselected_facet_items(code, limit)" class="facet-box">
<h2>${title(code)}</h2>
<ul class="facet-options">
- <li py:for="name, count in h.facet_items(code, limit=limit)"
- py:if="not (code, name) in c.fields">
- <a href="${c.drill_down_url(**{code: name})}">
- <span py:if="'format' in code.lower()">${h.icon(h.format_icon(name))}</span>
- ${label(name)}</a>${count_label(count)}
+ <li py:for="facet_item in h.unselected_facet_items(code, limit)"
+ py:if="not (code, facet_item.name) in c.fields">
+ <a href="${c.drill_down_url(**{code: facet_item.name})}">
+ <span py:if="'format' in code.lower()">${h.icon(h.format_icon(facet_item.name))}</span>
+ ${label(facet_item.name)}</a>${count_label(facet_item['count'])}
</li>
</ul>
- <p py:if="not len(h.facet_items(code, limit=limit))">${if_empty}</p>
+ <p py:if="not h.unselected_facet_items(code, limit)">${if_empty}</p>
</div>
</py:def>
+<!--
+DEPRECATED. Provided only for backward compatibility with existing plugins.
+ Use `facet_li` instead.
+
+Construct a possibly empty list of <li> elements containing links to further
+search results
+
+This is different from `facet_sidebar` and `facet_div` (above) in that it
+requires the caller to wrap up the resulting <li> elements in whatever dom
+element they need. But it *does* allow for filters to displayed in a
+hierarchy.
+
+label
+ Renders the human-readable label for each facet value.
+ If defined, this should be a callable that accepts a facet_item's name as a
+ string.
+
+if_empty
+ If if_empty is not None and there are no facets to filter on, then a single
+ <li> element is generated, with the text specified by if_empty
+
+-->
<py:def function="facet_list_items(code, limit=5, label=lambda n: n, if_empty=None)">
- <!-- Creates a possibly empty list of <li> elements containing links to further search results
-
- This is different from the above function in that it requires the caller to wrap
- up the resulting <li> elements in whatever dom element they need. But it does allow for
- filters to displayed in a hierarchy.
-
- If if_empty is not None and there are no facets to filter on, then a single <li> element
- is generated, with the text specified by if_empty
- -->
- <li py:if="if_empty and len(h.facet_items(code, limit=limit)) == 0">${if_empty}</li>
- <li py:for="name, count in h.facet_items(code, limit=limit)"
- py:if="not (code, name) in c.fields">
- <a href="${c.drill_down_url(**{code: name})}">${label(name)}</a> (${count})
- </li>
+
+ <?python
+ import logging
+ logging.getLogger('ckan.templates.facets').warning("Deprecated function:ckan/templates/facets.html:facet_list_items()")
+ ?>
+ <li py:if="if_empty and not h.unselected_facet_items(code, limit)">${if_empty}</li>
+ <li py:for="facet_item in h.unselected_facet_items(code, limit)">
+ <a href="${c.drill_down_url(**{code: facet_item.name})}">${label(facet_item.name)}</a> (facet_item['count'])
+ </li>
</py:def>
<py:def function="field_list()">
<div class="filter-list" py:if="c.fields">
<div class="filter-entry" py:for="(field, value) in c.fields">
<span class="name">${h.facet_title(field)}</span>
- <span class="value">${value}</span>
+ <span class="value" py:choose="">
+ <py:when test="c.translated_fields and c.translated_fields.has_key((field,value))">
+ ${c.translated_fields[(field,value)]}
+ </py:when>
+ <py:otherwise>
+ ${value}
+ </py:otherwise>
+ </span>
<a href="${c.remove_field(field, value)}">
${h.icon('unfilter')}
</a>
diff --git a/ckan/templates/group/read.html b/ckan/templates/group/read.html
index f3c6b9f..7886bdd 100644
--- a/ckan/templates/group/read.html
+++ b/ckan/templates/group/read.html
@@ -4,8 +4,9 @@
py:strip="">
<xi:include href="../facets.html" />
- <py:def function="page_title">${c.group.display_name}</py:def>
- <py:def function="page_heading">${c.group.display_name}</py:def>
+
+ <py:def function="page_title">${c.group_dict.display_name}</py:def>
+ <py:def function="page_heading">${c.group_dict.display_name}</py:def>
<py:if test="c.group.image_url">
<py:def function="page_logo">${c.group.image_url}</py:def>
</py:if>
@@ -24,8 +25,8 @@
</py:if>
</ul>
</li>
- ${facet_sidebar('tags')}
- ${facet_sidebar('res_format')}
+ ${facet_div('tags', 'Tags')}
+ ${facet_div('res_format', 'Resource Formats')}
</py:match>
<py:match path="content">
diff --git a/ckan/templates/package/search.html b/ckan/templates/package/search.html
index a73bfa2..e503295 100644
--- a/ckan/templates/package/search.html
+++ b/ckan/templates/package/search.html
@@ -20,10 +20,10 @@
<a href="${h.url_for(controller='package', action='new', id=None)}">Register it now</a>.
</p>
</li>
-
- ${facet_sidebar('tags')}
- ${facet_sidebar('res_format')}
- ${facet_sidebar('groups', label=h.group_name_to_title)}
+
+ ${facet_div('tags', 'Tags')}
+ ${facet_div('res_format', 'Resource Formats')}
+ ${facet_div('groups', 'Groups')}
<li class="widget-container widget_text">
<h4>Other access</h4>
diff --git a/ckan/templates/tag/index.html b/ckan/templates/tag/index.html
index 0452d34..17292e9 100644
--- a/ckan/templates/tag/index.html
+++ b/ckan/templates/tag/index.html
@@ -27,7 +27,7 @@
</p>
${c.page.pager(q=c.q)}
- ${tag_list_from_name(c.page.items)}
+ ${tag_list(c.page.items)}
${c.page.pager(q=c.q)}
<p py:if="c.q">
diff --git a/ckan/templates/tag/read.html b/ckan/templates/tag/read.html
index 24997ac..65256f0 100644
--- a/ckan/templates/tag/read.html
+++ b/ckan/templates/tag/read.html
@@ -3,11 +3,11 @@
xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">
- <py:def function="page_title">${c.tag['name']} - Tags</py:def>
- <py:def function="page_heading">Tag: ${c.tag['name']}</py:def>
+ <py:def function="page_title">${c.tag['display_name']} - Tags</py:def>
+ <py:def function="page_heading">Tag: ${c.tag['display_name']}</py:def>
<div py:match="content">
- <p i18n:msg="count, tagname">There are ${len(c.tag['packages'])} datasets tagged with <strong>${c.tag['name']}</strong>:</p>
+ <p i18n:msg="count, tagname">There are ${len(c.tag['packages'])} datasets tagged with <strong>${c.tag['display_name']}</strong>:</p>
${package_list_from_dict(c.tag['packages'])}
</div>
diff --git a/ckan/tests/functional/test_cors.py b/ckan/tests/functional/test_cors.py
index daa4d4e..b69ff1e 100644
--- a/ckan/tests/functional/test_cors.py
+++ b/ckan/tests/functional/test_cors.py
@@ -18,6 +18,6 @@ def test_headers(self):
headers = dict(out.headers)
print headers
assert headers['Access-Control-Allow-Origin'] == '*'
- assert headers['Access-Control-Allow-Methods'] == "POST, PUT, GET, DELETE"
- assert headers['Access-Control-Allow-Headers'] == "X-CKAN-API-KEY, Content-Type"
+ assert headers['Access-Control-Allow-Methods'] == "POST, PUT, GET, DELETE, OPTIONS"
+ assert headers['Access-Control-Allow-Headers'] == "X-CKAN-API-KEY, Authorization, Content-Type"
diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py
index c6ceb0b..09342d2 100644
--- a/ckan/tests/lib/test_dictization.py
+++ b/ckan/tests/lib/test_dictization.py
@@ -102,9 +102,13 @@ def setup_class(cls):
u'webstore_last_updated': None,
u'webstore_url': None}],
'state': u'active',
- 'tags': [{'name': u'Flexible \u30a1', 'state': u'active'},
- {'name': u'russian', 'state': u'active'},
- {'name': u'tolstoy', 'state': u'active'}],
+ 'tags': [{'name': u'Flexible \u30a1',
+ 'display_name': u'Flexible \u30a1',
+ 'state': u'active'},
+ {'name': u'russian', 'display_name': u'russian',
+ 'state': u'active'},
+ {'name': u'tolstoy', 'display_name': u'tolstoy',
+ 'state': u'active'}],
'title': u'A Novel By Tolstoy',
'url': u'http://www.annakarenina.com',
'version': u'0.7a'}
@@ -678,6 +682,7 @@ def test_13_get_package_in_past(self):
second_dictized['name'] = u'annakarenina_changed2'
second_dictized['resources'][0]['url'] = u'new_url2'
second_dictized['tags'][0]['name'] = u'new_tag'
+ second_dictized['tags'][0]['display_name'] = u'new_tag'
second_dictized['extras'][0]['value'] = u'"new_value"'
second_dictized['state'] = 'pending'
@@ -705,7 +710,7 @@ def test_13_get_package_in_past(self):
u'webstore_last_updated': None,
u'webstore_url': None})
- third_dictized['tags'].insert(1, {'name': u'newnew_tag', 'state': 'active'})
+ third_dictized['tags'].insert(1, {'name': u'newnew_tag', 'display_name': u'newnew_tag', 'state': 'active'})
third_dictized['extras'].insert(0, {'key': 'david',
'value': u'"new_value"',
'state': u'active'})
@@ -927,8 +932,9 @@ def test_16_group_dictized(self):
result = self.remove_changable_columns(group_dictized)
result['packages'] = sorted(result['packages'], key=lambda x: x['name'])
- assert result == expected, pformat(result)
-
+ assert_equal(sorted(result.keys()), sorted(expected.keys()))
+ for key in result:
+ assert_equal(sorted(result[key]), sorted(expected[key]))
def test_17_group_apis_to_dict(self):
diff --git a/ckan/tests/logic/__init__.py b/ckan/tests/logic/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py
index 8949ab9..a3294f1 100644
--- a/ckan/tests/logic/test_action.py
+++ b/ckan/tests/logic/test_action.py
@@ -1542,6 +1542,27 @@ def test_38_user_role_bulk_update(self):
{'domain_object': anna.id})
assert_equal(results['roles'], roles_after['roles'])
+ def test_40_task_resource_status(self):
+
+ import ckan.lib.celery_app as celery_app
+ backend = celery_app.celery.backend
+ ##This creates the database tables as a side effect, can not see another way
+ ##to make tables unless you actually create a task.
+ celery_result_session = backend.ResultSession()
+
+ ## need to do inserts as setting up an embedded celery is too much for these tests
+ model.Session.connection().execute(
+ '''INSERT INTO task_status (id, entity_id, entity_type, task_type, key, value, state, error, last_updated) VALUES ('5753adae-cd0d-4327-915d-edd832d1c9a3', '749cdcf2-3fc8-44ae-aed0-5eff8cc5032c', 'resource', 'qa', 'celery_task_id', '51f2105d-85b1-4393-b821-ac11475919d9', NULL, '', '2012-04-20 21:32:45.553986');
+ INSERT INTO celery_taskmeta (id, task_id, status, result, date_done, traceback) VALUES (2, '51f2105d-85b1-4393-b821-ac11475919d9', 'FAILURE', '52e', '2012-04-20 21:33:01.622557', 'Traceback')'''
+ )
+ model.Session.commit()
+ res = self.app.post('/api/action/resource_status_show',
+ params=json.dumps({'id': '749cdcf2-3fc8-44ae-aed0-5eff8cc5032c'}),
+ status=200)
+
+ assert json.loads(res.body) == {"help": None, "success": True, "result": [{"status": "FAILURE", "entity_id": "749cdcf2-3fc8-44ae-aed0-5eff8cc5032c", "task_type": "qa", "last_updated": "2012-04-20T21:32:45.553986", "date_done": "2012-04-20T21:33:01.622557", "entity_type": "resource", "traceback": "Traceback", "value": "51f2105d-85b1-4393-b821-ac11475919d9", "state": None, "key": "celery_task_id", "error": "", "id": "5753adae-cd0d-4327-915d-edd832d1c9a3"}]}
+
+
class TestActionTermTranslation(WsgiAppCase):
@classmethod
@@ -1594,7 +1615,7 @@ def test_1_update_single(self):
assert json.loads(res.body)['success']
- postparams = '%s=1' % json.dumps({"term" : "moo"})
+ postparams = '%s=1' % json.dumps({"terms" : ["moo"]})
res = self.app.post('/api/action/term_translation_show', params=postparams,
extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
@@ -1628,7 +1649,7 @@ def test_2_update_many(self):
assert json.loads(res.body)['result']['success'] == '3 rows updated', json.loads(res.body)
- postparams = '%s=1' % json.dumps({"term" : "many"})
+ postparams = '%s=1' % json.dumps({"terms" : ["many"]})
res = self.app.post('/api/action/term_translation_show', params=postparams,
extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
status=200)
diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py
new file mode 100644
index 0000000..a4696bd
--- /dev/null
+++ b/ckanext/multilingual/plugin.py
@@ -0,0 +1,276 @@
+import sets
+import ckan
+from ckan.plugins import SingletonPlugin, implements, IPackageController
+from ckan.plugins import IGroupController, ITagController
+import pylons
+import ckan.logic.action.get as action_get
+from pylons import config
+
+LANGS = ['en', 'fr', 'de', 'es', 'it', 'nl', 'ro', 'pt', 'pl']
+
+def translate_data_dict(data_dict):
+ '''Return the given dict (e.g. a dataset dict) with as many of its fields
+ as possible translated into the desired or the fallback language.
+
+ '''
+ desired_lang_code = pylons.request.environ['CKAN_LANG']
+ fallback_lang_code = pylons.config.get('ckan.locale_default', 'en')
+
+ # Get a flattened copy of data_dict to do the translation on.
+ flattened = ckan.lib.navl.dictization_functions.flatten_dict(
+ data_dict)
+
+ # Get a simple flat list of all the terms to be translated, from the
+ # flattened data dict.
+ terms = sets.Set()
+ for (key, value) in flattened.items():
+ if value in (None, True, False):
+ continue
+ elif isinstance(value, basestring):
+ terms.add(value)
+ elif isinstance(value, int):
+ continue
+ else:
+ for item in value:
+ terms.add(item)
+
+ # Get the translations of all the terms (as a list of dictionaries).
+ translations = ckan.logic.action.get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': terms,
+ 'lang_codes': (desired_lang_code, fallback_lang_code)})
+
+ # Transform the translations into a more convenient structure.
+ desired_translations = {}
+ fallback_translations = {}
+ for translation in translations:
+ if translation['lang_code'] == desired_lang_code:
+ desired_translations[translation['term']] = (
+ translation['term_translation'])
+ else:
+ assert translation['lang_code'] == fallback_lang_code
+ fallback_translations[translation['term']] = (
+ translation['term_translation'])
+
+ # Make a copy of the flattened data dict with all the terms replaced by
+ # their translations, where available.
+ translated_flattened = {}
+ for (key, value) in flattened.items():
+
+ # Don't translate names that are used for form URLs.
+ if key == ('name',):
+ translated_flattened[key] = value
+ elif (key[0] in ('tags', 'groups') and len(key) == 3
+ and key[2] == 'name'):
+ translated_flattened[key] = value
+
+ elif value in (None, True, False):
+ # Don't try to translate values that aren't strings.
+ translated_flattened[key] = value
+
+ elif isinstance(value, basestring):
+ if desired_translations.has_key(value):
+ translated_flattened[key] = desired_translations[value]
+ else:
+ translated_flattened[key] = fallback_translations.get(
+ value, value)
+
+ elif isinstance(value, int):
+ translated_flattened[key] = value
+
+ else:
+ translated_value = []
+ for item in value:
+ if desired_translations.has_key(value):
+ translated_flattened[key] = desired_translations[value]
+ else:
+ translated_flattened[key] = fallback_translations.get(
+ value, value)
+ translated_flattened[key] = translated_value
+
+ # Finally unflatten and return the translated data dict.
+ translated_data_dict = (ckan.lib.navl.dictization_functions
+ .unflatten(translated_flattened))
+ return translated_data_dict
+
+KEYS_TO_IGNORE = ['state', 'revision_id', 'id', #title done seperately
+ 'metadata_created', 'metadata_modified', 'site_id']
+
+class MultilingualDataset(SingletonPlugin):
+ implements(IPackageController, inherit=True)
+
+ def before_index(self, search_data):
+
+ default_lang = search_data.get(
+ 'lang_code',
+ pylons.config.get('ckan.locale_default', 'en')
+ )
+
+ ## translate title
+ title = search_data.get('title')
+ search_data['title_' + default_lang] = title
+ title_translations = action_get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': [title],
+ 'lang_codes': LANGS})
+
+ for translation in title_translations:
+ title_field = 'title_' + translation['lang_code']
+ search_data[title_field] = translation['term_translation']
+
+ ## translate rest
+ all_terms = []
+ for key, value in search_data.iteritems():
+ if key in KEYS_TO_IGNORE or key.startswith('title'):
+ continue
+ if isinstance(value, list):
+ all_terms.extend(value)
+ elif value in (None, True, False):
+ continue
+ else:
+ all_terms.append(value)
+
+ field_translations = action_get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': all_terms,
+ 'lang_codes': LANGS})
+
+ text_field_items = dict(('text_' + lang, []) for lang in LANGS)
+
+ text_field_items['text_' + default_lang].extend(all_terms)
+
+ for translation in sorted(field_translations):
+ lang_field = 'text_' + translation['lang_code']
+ text_field_items[lang_field].append(translation['term_translation'])
+
+ for key, value in text_field_items.iteritems():
+ search_data[key] = ' '.join(value)
+
+ return search_data
+
+ def before_search(self, search_params):
+ lang_set = set(LANGS)
+ current_lang = pylons.request.environ['CKAN_LANG']
+ # fallback to default locale if locale not in suported langs
+ if not current_lang in lang_set:
+ current_lang = config.get('ckan.locale_default')
+ # fallback to english if default locale is not supported
+ if not current_lang in lang_set:
+ current_lang = 'en'
+ # treat current lang differenly so remove from set
+ lang_set.remove(current_lang)
+
+ # weight current lang more highly
+ query_fields = 'title_%s^8 text_%s^4' % (current_lang, current_lang)
+
+ for lang in lang_set:
+ query_fields += ' title_%s^2 text_%s' % (lang, lang)
+
+ search_params['qf'] = query_fields
+
+ return search_params
+
+ def after_search(self, search_results, search_params):
+
+ # Translate the unselected search facets.
+ facets = search_results.get('search_facets')
+ if not facets:
+ return search_results
+
+ desired_lang_code = pylons.request.environ['CKAN_LANG']
+ fallback_lang_code = pylons.config.get('ckan.locale_default', 'en')
+
+ # Look up translations for all of the facets in one db query.
+ terms = sets.Set()
+ for facet in facets.values():
+ for item in facet['items']:
+ terms.add(item['display_name'])
+ translations = ckan.logic.action.get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': terms,
+ 'lang_codes': (desired_lang_code, fallback_lang_code)})
+
+ # Replace facet display names with translated ones.
+ for facet in facets.values():
+ for item in facet['items']:
+ matching_translations = [translation for
+ translation in translations
+ if translation['term'] == item['display_name']
+ and translation['lang_code'] == desired_lang_code]
+ if not matching_translations:
+ matching_translations = [translation for
+ translation in translations
+ if translation['term'] == item['display_name']
+ and translation['lang_code'] == fallback_lang_code]
+ if matching_translations:
+ assert len(matching_translations) == 1
+ item['display_name'] = (
+ matching_translations[0]['term_translation'])
+
+ return search_results
+
+ def before_view(self, dataset_dict):
+
+ # Translate any selected search facets (e.g. if we are rendering a
+ # group read page or the dataset index page): lookup translations of
+ # all the terms in c.fields (c.fields contains the selected facets)
+ # and save them in c.translated_fields where the templates can
+ # retrieve them later.
+ c = pylons.c
+ desired_lang_code = pylons.request.environ['CKAN_LANG']
+ fallback_lang_code = pylons.config.get('ckan.locale_default', 'en')
+ terms = [value for param, value in c.fields]
+ translations = ckan.logic.action.get.term_translation_show(
+ {'model': ckan.model},
+ {'terms': terms,
+ 'lang_codes': (desired_lang_code, fallback_lang_code)})
+ c.translated_fields = {}
+ for param, value in c.fields:
+ matching_translations = [translation for translation in
+ translations if translation['term'] == value and
+ translation['lang_code'] == desired_lang_code]
+ if not matching_translations:
+ matching_translations = [translation for translation in
+ translations if translation['term'] == value and
+ translation['lang_code'] == fallback_lang_code]
+ if matching_translations:
+ assert len(matching_translations) == 1
+ translation = matching_translations[0]['term_translation']
+ c.translated_fields[(param, value)] = translation
+
+ # Now translate the fields of the dataset itself.
+ return translate_data_dict(dataset_dict)
+
+class MultilingualGroup(SingletonPlugin):
+ '''The MultilingualGroup plugin translates group names and other group
+ fields on group read pages and on the group index page.
+
+ For example on the page /de/group/david the title "Dave's Books" at the
+ top of the page might be translated to "Dave's Bucher".
+
+ Datasets are also shown on group pages, but these are translated by the
+ MultilingualDataset plugin.
+
+ '''
+ implements(IGroupController, inherit=True)
+
+ def before_view(self, data_dict):
+ translated_data_dict = translate_data_dict(data_dict)
+ return translated_data_dict
+
+class MultilingualTag(SingletonPlugin):
+ '''The MultilingualTag plugin translates tag names on tag read pages and
+ on the tag index page.
+
+ For example on the page /de/tag/tolstoy the title "Tag: tolstoy" at the
+ top of the page might be translated to "Tag: Tolstoi".
+
+ Datasets are also shown on tag pages, but these are translated by the
+ MultilingualDataset plugin.
+
+ '''
+ implements(ITagController, inherit=True)
+
+ def before_view(self, data_dict):
+ translated_data_dict = translate_data_dict(data_dict)
+ return translated_data_dict
diff --git a/ckanext/multilingual/solr/dutch_stop.txt b/ckanext/multilingual/solr/dutch_stop.txt
new file mode 100644
index 0000000..f4d61f5
--- /dev/null
+++ b/ckanext/multilingual/solr/dutch_stop.txt
@@ -0,0 +1,117 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A Dutch stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | This is a ranked list (commonest to rarest) of stopwords derived from
+ | a large sample of Dutch text.
+
+ | Dutch stop words frequently exhibit homonym clashes. These are indicated
+ | clearly below.
+
+de | the
+en | and
+van | of, from
+ik | I, the ego
+te | (1) chez, at etc, (2) to, (3) too
+dat | that, which
+die | that, those, who, which
+in | in, inside
+een | a, an, one
+hij | he
+het | the, it
+niet | not, nothing, naught
+zijn | (1) to be, being, (2) his, one's, its
+is | is
+was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river
+op | on, upon, at, in, up, used up
+aan | on, upon, to (as dative)
+met | with, by
+als | like, such as, when
+voor | (1) before, in front of, (2) furrow
+had | had, past tense all persons sing. of 'hebben' (have)
+er | there
+maar | but, only
+om | round, about, for etc
+hem | him
+dan | then
+zou | should/would, past tense all persons sing. of 'zullen'
+of | or, whether, if
+wat | what, something, anything
+mijn | possessive and noun 'mine'
+men | people, 'one'
+dit | this
+zo | so, thus, in this way
+door | through by
+over | over, across
+ze | she, her, they, them
+zich | oneself
+bij | (1) a bee, (2) by, near, at
+ook | also, too
+tot | till, until
+je | you
+mij | me
+uit | out of, from
+der | Old Dutch form of 'van der' still found in surnames
+daar | (1) there, (2) because
+haar | (1) her, their, them, (2) hair
+naar | (1) unpleasant, unwell etc, (2) towards, (3) as
+heb | present first person sing. of 'to have'
+hoe | how, why
+heeft | present third person sing. of 'to have'
+hebben | 'to have' and various parts thereof
+deze | this
+u | you
+want | (1) for, (2) mitten, (3) rigging
+nog | yet, still
+zal | 'shall', first and third person sing. of verb 'zullen' (will)
+me | me
+zij | she, they
+nu | now
+ge | 'thou', still used in Belgium and south Netherlands
+geen | none
+omdat | because
+iets | something, somewhat
+worden | to become, grow, get
+toch | yet, still
+al | all, every, each
+waren | (1) 'were' (2) to wander, (3) wares, (3)
+veel | much, many
+meer | (1) more, (2) lake
+doen | to do, to make
+toen | then, when
+moet | noun 'spot/mote' and present form of 'to must'
+ben | (1) am, (2) 'are' in interrogative second person singular of 'to be'
+zonder | without
+kan | noun 'can' and present form of 'to be able'
+hun | their, them
+dus | so, consequently
+alles | all, everything, anything
+onder | under, beneath
+ja | yes, of course
+eens | once, one day
+hier | here
+wie | who
+werd | imperfect third person sing. of 'become'
+altijd | always
+doch | yet, but etc
+wordt | present third person sing. of 'become'
+wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans
+kunnen | to be able
+ons | us/our
+zelf | self
+tegen | against, towards, at
+na | after, near
+reeds | already
+wil | (1) present tense of 'want', (2) 'will', noun, (3) fender
+kon | could; past tense of 'to be able'
+niets | nothing
+uw | your
+iemand | somebody
+geweest | been; past participle of 'be'
+andere | other
diff --git a/ckanext/multilingual/solr/english_stop.txt b/ckanext/multilingual/solr/english_stop.txt
new file mode 100644
index 0000000..821c03e
--- /dev/null
+++ b/ckanext/multilingual/solr/english_stop.txt
@@ -0,0 +1,317 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/english/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | An English stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | Many of the forms below are quite rare (e.g. "yourselves") but included for
+ | completeness.
+
+ | PRONOUNS FORMS
+ | 1st person sing
+
+i | subject, always in upper case of course
+
+me | object
+my | possessive adjective
+ | the possessive pronoun `mine' is best suppressed, because of the
+ | sense of coal-mine etc.
+myself | reflexive
+ | 1st person plural
+we | subject
+
+| us | object
+ | care is required here because US = United States. It is usually
+ | safe to remove it if it is in lower case.
+our | possessive adjective
+ours | possessive pronoun
+ourselves | reflexive
+ | second person (archaic `thou' forms not included)
+you | subject and object
+your | possessive adjective
+yours | possessive pronoun
+yourself | reflexive (singular)
+yourselves | reflexive (plural)
+ | third person singular
+he | subject
+him | object
+his | possessive adjective and pronoun
+himself | reflexive
+
+she | subject
+her | object and possessive adjective
+hers | possessive pronoun
+herself | reflexive
+
+it | subject and object
+its | possessive adjective
+itself | reflexive
+ | third person plural
+they | subject
+them | object
+their | possessive adjective
+theirs | possessive pronoun
+themselves | reflexive
+ | other forms (demonstratives, interrogatives)
+what
+which
+who
+whom
+this
+that
+these
+those
+
+ | VERB FORMS (using F.R. Palmer's nomenclature)
+ | BE
+am | 1st person, present
+is | -s form (3rd person, present)
+are | present
+was | 1st person, past
+were | past
+be | infinitive
+been | past participle
+being | -ing form
+ | HAVE
+have | simple
+has | -s form
+had | past
+having | -ing form
+ | DO
+do | simple
+does | -s form
+did | past
+doing | -ing form
+
+ | The forms below are, I believe, best omitted, because of the significant
+ | homonym forms:
+
+ | He made a WILL
+ | old tin CAN
+ | merry month of MAY
+ | a smell of MUST
+ | fight the good fight with all thy MIGHT
+
+ | would, could, should, ought might however be included
+
+ | | AUXILIARIES
+ | | WILL
+ |will
+
+would
+
+ | | SHALL
+ |shall
+
+should
+
+ | | CAN
+ |can
+
+could
+
+ | | MAY
+ |may
+ |might
+ | | MUST
+ |must
+ | | OUGHT
+
+ought
+
+ | COMPOUND FORMS, increasingly encountered nowadays in 'formal' writing
+ | pronoun + verb
+
+i'm
+you're
+he's
+she's
+it's
+we're
+they're
+i've
+you've
+we've
+they've
+i'd
+you'd
+he'd
+she'd
+we'd
+they'd
+i'll
+you'll
+he'll
+she'll
+we'll
+they'll
+
+ | verb + negation
+
+isn't
+aren't
+wasn't
+weren't
+hasn't
+haven't
+hadn't
+doesn't
+don't
+didn't
+
+ | auxiliary + negation
+
+won't
+wouldn't
+shan't
+shouldn't
+can't
+cannot
+couldn't
+mustn't
+
+ | miscellaneous forms
+
+let's
+that's
+who's
+what's
+here's
+there's
+when's
+where's
+why's
+how's
+
+ | rarer forms
+
+ | daren't needn't
+
+ | doubtful forms
+
+ | oughtn't mightn't
+
+ | ARTICLES
+a
+an
+the
+
+ | THE REST (Overlap among prepositions, conjunctions, adverbs etc is so
+ | high, that classification is pointless.)
+and
+but
+if
+or
+because
+as
+until
+while
+
+of
+at
+by
+for
+with
+about
+against
+between
+into
+through
+during
+before
+after
+above
+below
+to
+from
+up
+down
+in
+out
+on
+off
+over
+under
+
+again
+further
+then
+once
+
+here
+there
+when
+where
+why
+how
+
+all
+any
+both
+each
+few
+more
+most
+other
+some
+such
+
+no
+nor
+not
+only
+own
+same
+so
+than
+too
+very
+
+ | Just for the record, the following words are among the commonest in English
+
+ | one
+ | every
+ | least
+ | less
+ | many
+ | now
+ | ever
+ | never
+ | say
+ | says
+ | said
+ | also
+ | get
+ | go
+ | goes
+ | just
+ | made
+ | make
+ | put
+ | see
+ | seen
+ | whether
+ | like
+ | well
+ | back
+ | even
+ | still
+ | way
+ | take
+ | since
+ | another
+ | however
+ | two
+ | three
+ | four
+ | five
+ | first
+ | second
+ | new
+ | old
+ | high
+ | long
diff --git a/ckanext/multilingual/solr/fr_elision.txt b/ckanext/multilingual/solr/fr_elision.txt
new file mode 100644
index 0000000..4e855c1
--- /dev/null
+++ b/ckanext/multilingual/solr/fr_elision.txt
@@ -0,0 +1,8 @@
+c
+d
+j
+l
+m
+n
+s
+t
diff --git a/ckanext/multilingual/solr/french_stop.txt b/ckanext/multilingual/solr/french_stop.txt
new file mode 100644
index 0000000..c00837e
--- /dev/null
+++ b/ckanext/multilingual/solr/french_stop.txt
@@ -0,0 +1,183 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A French stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+au | a + le
+aux | a + les
+avec | with
+ce | this
+ces | these
+dans | with
+de | of
+des | de + les
+du | de + le
+elle | she
+en | `of them' etc
+et | and
+eux | them
+il | he
+je | I
+la | the
+le | the
+leur | their
+lui | him
+ma | my (fem)
+mais | but
+me | me
+même | same; as in moi-même (myself) etc
+mes | me (pl)
+moi | me
+mon | my (masc)
+ne | not
+nos | our (pl)
+notre | our
+nous | we
+on | one
+ou | where
+par | by
+pas | not
+pour | for
+qu | que before vowel
+que | that
+qui | who
+sa | his, her (fem)
+se | oneself
+ses | his (pl)
+son | his, her (masc)
+sur | on
+ta | thy (fem)
+te | thee
+tes | thy (pl)
+toi | thee
+ton | thy (masc)
+tu | thou
+un | a
+une | a
+vos | your (pl)
+votre | your
+vous | you
+
+ | single letter forms
+
+c | c'
+d | d'
+j | j'
+l | l'
+à | to, at
+m | m'
+n | n'
+s | s'
+t | t'
+y | there
+
+ | forms of être (not including the infinitive):
+été
+étée
+étées
+étés
+étant
+suis
+es
+est
+sommes
+êtes
+sont
+serai
+seras
+sera
+serons
+serez
+seront
+serais
+serait
+serions
+seriez
+seraient
+étais
+était
+étions
+étiez
+étaient
+fus
+fut
+fûmes
+fûtes
+furent
+sois
+soit
+soyons
+soyez
+soient
+fusse
+fusses
+fût
+fussions
+fussiez
+fussent
+
+ | forms of avoir (not including the infinitive):
+ayant
+eu
+eue
+eues
+eus
+ai
+as
+avons
+avez
+ont
+aurai
+auras
+aura
+aurons
+aurez
+auront
+aurais
+aurait
+aurions
+auriez
+auraient
+avais
+avait
+avions
+aviez
+avaient
+eut
+eûmes
+eûtes
+eurent
+aie
+aies
+ait
+ayons
+ayez
+aient
+eusse
+eusses
+eût
+eussions
+eussiez
+eussent
+
+ | Later additions (from Jean-Christophe Deschamps)
+ceci | this
+celà | that
+cet | this
+cette | this
+ici | here
+ils | they
+les | the (pl)
+leurs | their (pl)
+quel | which
+quels | which
+quelle | which
+quelles | which
+sans | without
+soi | oneself
+
diff --git a/ckanext/multilingual/solr/german_stop.txt b/ckanext/multilingual/solr/german_stop.txt
new file mode 100644
index 0000000..f770384
--- /dev/null
+++ b/ckanext/multilingual/solr/german_stop.txt
@@ -0,0 +1,292 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A German stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | The number of forms in this list is reduced significantly by passing it
+ | through the German stemmer.
+
+
+aber | but
+
+alle | all
+allem
+allen
+aller
+alles
+
+als | than, as
+also | so
+am | an + dem
+an | at
+
+ander | other
+andere
+anderem
+anderen
+anderer
+anderes
+anderm
+andern
+anderr
+anders
+
+auch | also
+auf | on
+aus | out of
+bei | by
+bin | am
+bis | until
+bist | art
+da | there
+damit | with it
+dann | then
+
+der | the
+den
+des
+dem
+die
+das
+
+daß | that
+
+derselbe | the same
+derselben
+denselben
+desselben
+demselben
+dieselbe
+dieselben
+dasselbe
+
+dazu | to that
+
+dein | thy
+deine
+deinem
+deinen
+deiner
+deines
+
+denn | because
+
+derer | of those
+dessen | of him
+
+dich | thee
+dir | to thee
+du | thou
+
+dies | this
+diese
+diesem
+diesen
+dieser
+dieses
+
+
+doch | (several meanings)
+dort | (over) there
+
+
+durch | through
+
+ein | a
+eine
+einem
+einen
+einer
+eines
+
+einig | some
+einige
+einigem
+einigen
+einiger
+einiges
+
+einmal | once
+
+er | he
+ihn | him
+ihm | to him
+
+es | it
+etwas | something
+
+euer | your
+eure
+eurem
+euren
+eurer
+eures
+
+für | for
+gegen | towards
+gewesen | p.p. of sein
+hab | have
+habe | have
+haben | have
+hat | has
+hatte | had
+hatten | had
+hier | here
+hin | there
+hinter | behind
+
+ich | I
+mich | me
+mir | to me
+
+
+ihr | you, to her
+ihre
+ihrem
+ihren
+ihrer
+ihres
+euch | to you
+
+im | in + dem
+in | in
+indem | while
+ins | in + das
+ist | is
+
+jede | each, every
+jedem
+jeden
+jeder
+jedes
+
+jene | that
+jenem
+jenen
+jener
+jenes
+
+jetzt | now
+kann | can
+
+kein | no
+keine
+keinem
+keinen
+keiner
+keines
+
+können | can
+könnte | could
+machen | do
+man | one
+
+manche | some, many a
+manchem
+manchen
+mancher
+manches
+
+mein | my
+meine
+meinem
+meinen
+meiner
+meines
+
+mit | with
+muss | must
+musste | had to
+nach | to(wards)
+nicht | not
+nichts | nothing
+noch | still, yet
+nun | now
+nur | only
+ob | whether
+oder | or
+ohne | without
+sehr | very
+
+sein | his
+seine
+seinem
+seinen
+seiner
+seines
+
+selbst | self
+sich | herself
+
+sie | they, she
+ihnen | to them
+
+sind | are
+so | so
+
+solche | such
+solchem
+solchen
+solcher
+solches
+
+soll | shall
+sollte | should
+sondern | but
+sonst | else
+über | over
+um | about, around
+und | and
+
+uns | us
+unse
+unsem
+unsen
+unser
+unses
+
+unter | under
+viel | much
+vom | von + dem
+von | from
+vor | before
+während | while
+war | was
+waren | were
+warst | wast
+was | what
+weg | away, off
+weil | because
+weiter | further
+
+welche | which
+welchem
+welchen
+welcher
+welches
+
+wenn | when
+werde | will
+werden | will
+wie | how
+wieder | again
+will | want
+wir | we
+wird | will
+wirst | willst
+wo | where
+wollen | want
+wollte | wanted
+würde | would
+würden | would
+zu | to
+zum | zu + dem
+zur | zu + der
+zwar | indeed
+zwischen | between
+
diff --git a/ckanext/multilingual/solr/greek_stopwords.txt b/ckanext/multilingual/solr/greek_stopwords.txt
new file mode 100644
index 0000000..1a08d31
--- /dev/null
+++ b/ckanext/multilingual/solr/greek_stopwords.txt
@@ -0,0 +1,76 @@
+# Lucene Greek Stopwords list
+ο
+η
+το
+οι
+τα
+του
+τησ
+των
+τον
+την
+και
+κι
+κ
+ειμαι
+εισαι
+ειναι
+ειμαστε
+ειστε
+στο
+στον
+στη
+στην
+μα
+αλλα
+απο
+για
+προσ
+με
+σε
+ωσ
+παρα
+αντι
+κατα
+μετα
+θα
+να
+δε
+δεν
+μη
+μην
+επι
+ενω
+εαν
+αν
+τοτε
+που
+πωσ
+ποιοσ
+ποια
+ποιο
+ποιοι
+ποιεσ
+ποιων
+ποιουσ
+αυτοσ
+αυτη
+αυτο
+αυτοι
+αυτων
+αυτουσ
+αυτεσ
+αυτα
+εκεινοσ
+εκεινη
+εκεινο
+εκεινοι
+εκεινεσ
+εκεινα
+εκεινων
+εκεινουσ
+οπωσ
+ομωσ
+ισωσ
+οσο
+οτι
diff --git a/ckanext/multilingual/solr/italian_stop.txt b/ckanext/multilingual/solr/italian_stop.txt
new file mode 100644
index 0000000..4cb5b08
--- /dev/null
+++ b/ckanext/multilingual/solr/italian_stop.txt
@@ -0,0 +1,301 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | An Italian stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ad | a (to) before vowel
+al | a + il
+allo | a + lo
+ai | a + i
+agli | a + gli
+all | a + l'
+agl | a + gl'
+alla | a + la
+alle | a + le
+con | with
+col | con + il
+coi | con + i (forms collo, cogli etc are now very rare)
+da | from
+dal | da + il
+dallo | da + lo
+dai | da + i
+dagli | da + gli
+dall | da + l'
+dagl | da + gll'
+dalla | da + la
+dalle | da + le
+di | of
+del | di + il
+dello | di + lo
+dei | di + i
+degli | di + gli
+dell | di + l'
+degl | di + gl'
+della | di + la
+delle | di + le
+in | in
+nel | in + el
+nello | in + lo
+nei | in + i
+negli | in + gli
+nell | in + l'
+negl | in + gl'
+nella | in + la
+nelle | in + le
+su | on
+sul | su + il
+sullo | su + lo
+sui | su + i
+sugli | su + gli
+sull | su + l'
+sugl | su + gl'
+sulla | su + la
+sulle | su + le
+per | through, by
+tra | among
+contro | against
+io | I
+tu | thou
+lui | he
+lei | she
+noi | we
+voi | you
+loro | they
+mio | my
+mia |
+miei |
+mie |
+tuo |
+tua |
+tuoi | thy
+tue |
+suo |
+sua |
+suoi | his, her
+sue |
+nostro | our
+nostra |
+nostri |
+nostre |
+vostro | your
+vostra |
+vostri |
+vostre |
+mi | me
+ti | thee
+ci | us, there
+vi | you, there
+lo | him, the
+la | her, the
+li | them
+le | them, the
+gli | to him, the
+ne | from there etc
+il | the
+un | a
+uno | a
+una | a
+ma | but
+ed | and
+se | if
+perché | why, because
+anche | also
+come | how
+dov | where (as dov')
+dove | where
+che | who, that
+chi | who
+cui | whom
+non | not
+più | more
+quale | who, that
+quanto | how much
+quanti |
+quanta |
+quante |
+quello | that
+quelli |
+quella |
+quelle |
+questo | this
+questi |
+questa |
+queste |
+si | yes
+tutto | all
+tutti | all
+
+ | single letter forms:
+
+a | at
+c | as c' for ce or ci
+e | and
+i | the
+l | as l'
+o | or
+
+ | forms of avere, to have (not including the infinitive):
+
+ho
+hai
+ha
+abbiamo
+avete
+hanno
+abbia
+abbiate
+abbiano
+avrò
+avrai
+avrà
+avremo
+avrete
+avranno
+avrei
+avresti
+avrebbe
+avremmo
+avreste
+avrebbero
+avevo
+avevi
+aveva
+avevamo
+avevate
+avevano
+ebbi
+avesti
+ebbe
+avemmo
+aveste
+ebbero
+avessi
+avesse
+avessimo
+avessero
+avendo
+avuto
+avuta
+avuti
+avute
+
+ | forms of essere, to be (not including the infinitive):
+sono
+sei
+è
+siamo
+siete
+sia
+siate
+siano
+sarò
+sarai
+sarà
+saremo
+sarete
+saranno
+sarei
+saresti
+sarebbe
+saremmo
+sareste
+sarebbero
+ero
+eri
+era
+eravamo
+eravate
+erano
+fui
+fosti
+fu
+fummo
+foste
+furono
+fossi
+fosse
+fossimo
+fossero
+essendo
+
+ | forms of fare, to do (not including the infinitive, fa, fat-):
+faccio
+fai
+facciamo
+fanno
+faccia
+facciate
+facciano
+farò
+farai
+farà
+faremo
+farete
+faranno
+farei
+faresti
+farebbe
+faremmo
+fareste
+farebbero
+facevo
+facevi
+faceva
+facevamo
+facevate
+facevano
+feci
+facesti
+fece
+facemmo
+faceste
+fecero
+facessi
+facesse
+facessimo
+facessero
+facendo
+
+ | forms of stare, to be (not including the infinitive):
+sto
+stai
+sta
+stiamo
+stanno
+stia
+stiate
+stiano
+starò
+starai
+starà
+staremo
+starete
+staranno
+starei
+staresti
+starebbe
+staremmo
+stareste
+starebbero
+stavo
+stavi
+stava
+stavamo
+stavate
+stavano
+stetti
+stesti
+stette
+stemmo
+steste
+stettero
+stessi
+stesse
+stessimo
+stessero
+stando
diff --git a/ckanext/multilingual/solr/polish_stop.txt b/ckanext/multilingual/solr/polish_stop.txt
new file mode 100644
index 0000000..167e9e0
--- /dev/null
+++ b/ckanext/multilingual/solr/polish_stop.txt
@@ -0,0 +1,186 @@
+# This file was created from the carrot2 project and is distributed under the BSD license.
+# See http://project.carrot2.org/license.html
+# Also see http://www.opensource.org/licenses/bsd-license.html
+# From trunk/core/carrot2-util-text/src-resources/stopwords.pl
+vol
+o.o.
+mgr
+godz
+zł
+www
+pl
+ul
+tel
+hab
+prof
+inż
+dr
+i
+u
+aby
+albo
+ale
+ani
+aż
+bardzo
+bez
+bo
+bowiem
+by
+byli
+bym
+był
+była
+było
+były
+być
+będzie
+będą
+chce
+choć
+co
+coraz
+coś
+czy
+czyli
+często
+dla
+do
+gdy
+gdyby
+gdyż
+gdzie
+go
+ich
+im
+inne
+iż
+ja
+jak
+jakie
+jako
+je
+jednak
+jednym
+jedynie
+jego
+jej
+jest
+jeszcze
+jeśli
+jeżeli
+już
+ją
+kiedy
+kilku
+kto
+która
+które
+którego
+której
+który
+których
+którym
+którzy
+lat
+lecz
+lub
+ma
+mają
+mamy
+mi
+miał
+mimo
+mnie
+mogą
+może
+można
+mu
+musi
+na
+nad
+nam
+nas
+nawet
+nic
+nich
+nie
+niej
+nim
+niż
+no
+nowe
+np
+nr
+o
+od
+ok
+on
+one
+oraz
+pan
+po
+pod
+ponad
+ponieważ
+poza
+przed
+przede
+przez
+przy
+raz
+razie
+roku
+również
+się
+sobie
+sposób
+swoje
+są
+ta
+tak
+takich
+takie
+także
+tam
+te
+tego
+tej
+temu
+ten
+teraz
+też
+to
+trzeba
+tu
+tych
+tylko
+tym
+tys
+tzw
+tę
+w
+we
+wie
+więc
+wszystko
+wśród
+właśnie
+z
+za
+zaś
+ze
+że
+żeby
+ii
+iii
+iv
+vi
+vii
+viii
+ix
+xi
+xii
+xiii
+xiv
+xv
diff --git a/ckanext/multilingual/solr/portuguese_stop.txt b/ckanext/multilingual/solr/portuguese_stop.txt
new file mode 100644
index 0000000..276c1b4
--- /dev/null
+++ b/ckanext/multilingual/solr/portuguese_stop.txt
@@ -0,0 +1,251 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A Portuguese stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+
+ | The following is a ranked list (commonest to rarest) of stopwords
+ | deriving from a large sample of text.
+
+ | Extra words have been added at the end.
+
+de | of, from
+a | the; to, at; her
+o | the; him
+que | who, that
+e | and
+do | de + o
+da | de + a
+em | in
+um | a
+para | for
+ | é from SER
+com | with
+não | not, no
+uma | a
+os | the; them
+no | em + o
+se | himself etc
+na | em + a
+por | for
+mais | more
+as | the; them
+dos | de + os
+como | as, like
+mas | but
+ | foi from SER
+ao | a + o
+ele | he
+das | de + as
+ | tem from TER
+à | a + a
+seu | his
+sua | her
+ou | or
+ | ser from SER
+quando | when
+muito | much
+ | há from HAV
+nos | em + os; us
+já | already, now
+ | está from EST
+eu | I
+também | also
+só | only, just
+pelo | per + o
+pela | per + a
+até | up to
+isso | that
+ela | he
+entre | between
+ | era from SER
+depois | after
+sem | without
+mesmo | same
+aos | a + os
+ | ter from TER
+seus | his
+quem | whom
+nas | em + as
+me | me
+esse | that
+eles | they
+ | estão from EST
+você | you
+ | tinha from TER
+ | foram from SER
+essa | that
+num | em + um
+nem | nor
+suas | her
+meu | my
+às | a + as
+minha | my
+ | têm from TER
+numa | em + uma
+pelos | per + os
+elas | they
+ | havia from HAV
+ | seja from SER
+qual | which
+ | será from SER
+nós | we
+ | tenho from TER
+lhe | to him, her
+deles | of them
+essas | those
+esses | those
+pelas | per + as
+este | this
+ | fosse from SER
+dele | of him
+
+ | other words. There are many contractions such as naquele = em+aquele,
+ | mo = me+o, but they are rare.
+ | Indefinite article plural forms are also rare.
+
+tu | thou
+te | thee
+vocês | you (plural)
+vos | you
+lhes | to them
+meus | my
+minhas
+teu | thy
+tua
+teus
+tuas
+nosso | our
+nossa
+nossos
+nossas
+
+dela | of her
+delas | of them
+
+esta | this
+estes | these
+estas | these
+aquele | that
+aquela | that
+aqueles | those
+aquelas | those
+isto | this
+aquilo | that
+
+ | forms of estar, to be (not including the infinitive):
+estou
+está
+estamos
+estão
+estive
+esteve
+estivemos
+estiveram
+estava
+estávamos
+estavam
+estivera
+estivéramos
+esteja
+estejamos
+estejam
+estivesse
+estivéssemos
+estivessem
+estiver
+estivermos
+estiverem
+
+ | forms of haver, to have (not including the infinitive):
+hei
+há
+havemos
+hão
+houve
+houvemos
+houveram
+houvera
+houvéramos
+haja
+hajamos
+hajam
+houvesse
+houvéssemos
+houvessem
+houver
+houvermos
+houverem
+houverei
+houverá
+houveremos
+houverão
+houveria
+houveríamos
+houveriam
+
+ | forms of ser, to be (not including the infinitive):
+sou
+somos
+são
+era
+éramos
+eram
+fui
+foi
+fomos
+foram
+fora
+fôramos
+seja
+sejamos
+sejam
+fosse
+fôssemos
+fossem
+for
+formos
+forem
+serei
+será
+seremos
+serão
+seria
+seríamos
+seriam
+
+ | forms of ter, to have (not including the infinitive):
+tenho
+tem
+temos
+tém
+tinha
+tínhamos
+tinham
+tive
+teve
+tivemos
+tiveram
+tivera
+tivéramos
+tenha
+tenhamos
+tenham
+tivesse
+tivéssemos
+tivessem
+tiver
+tivermos
+tiverem
+terei
+terá
+teremos
+terão
+teria
+teríamos
+teriam
diff --git a/ckanext/multilingual/solr/romanian_stop.txt b/ckanext/multilingual/solr/romanian_stop.txt
new file mode 100644
index 0000000..4fdee90
--- /dev/null
+++ b/ckanext/multilingual/solr/romanian_stop.txt
@@ -0,0 +1,233 @@
+# This file was created by Jacques Savoy and is distributed under the BSD license.
+# See http://members.unine.ch/jacques.savoy/clef/index.html.
+# Also see http://www.opensource.org/licenses/bsd-license.html
+acea
+aceasta
+această
+aceea
+acei
+aceia
+acel
+acela
+acele
+acelea
+acest
+acesta
+aceste
+acestea
+aceşti
+aceştia
+acolo
+acum
+ai
+aia
+aibă
+aici
+al
+ăla
+ale
+alea
+ălea
+altceva
+altcineva
+am
+ar
+are
+aş
+aşadar
+asemenea
+asta
+ăsta
+astăzi
+astea
+ăstea
+ăştia
+asupra
+aţi
+au
+avea
+avem
+aveţi
+azi
+bine
+bucur
+bună
+ca
+că
+căci
+când
+care
+cărei
+căror
+cărui
+cât
+câte
+câţi
+către
+câtva
+ce
+cel
+ceva
+chiar
+cînd
+cine
+cineva
+cît
+cîte
+cîţi
+cîtva
+contra
+cu
+cum
+cumva
+curând
+curînd
+da
+dă
+dacă
+dar
+datorită
+de
+deci
+deja
+deoarece
+departe
+deşi
+din
+dinaintea
+dintr
+dintre
+drept
+după
+ea
+ei
+el
+ele
+eram
+este
+eşti
+eu
+face
+fără
+fi
+fie
+fiecare
+fii
+fim
+fiţi
+iar
+ieri
+îi
+îl
+îmi
+împotriva
+în
+înainte
+înaintea
+încât
+încît
+încotro
+între
+întrucât
+întrucît
+îţi
+la
+lângă
+le
+li
+lîngă
+lor
+lui
+mă
+mâine
+mea
+mei
+mele
+mereu
+meu
+mi
+mine
+mult
+multă
+mulţi
+ne
+nicăieri
+nici
+nimeni
+nişte
+noastră
+noastre
+noi
+noştri
+nostru
+nu
+ori
+oricând
+oricare
+oricât
+orice
+oricînd
+oricine
+oricît
+oricum
+oriunde
+până
+pe
+pentru
+peste
+pînă
+poate
+pot
+prea
+prima
+primul
+prin
+printr
+sa
+să
+săi
+sale
+sau
+său
+se
+şi
+sînt
+sîntem
+sînteţi
+spre
+sub
+sunt
+suntem
+sunteţi
+ta
+tăi
+tale
+tău
+te
+ţi
+ţie
+tine
+toată
+toate
+tot
+toţi
+totuşi
+tu
+un
+una
+unde
+undeva
+unei
+unele
+uneori
+unor
+vă
+vi
+voastră
+voastre
+voi
+voştri
+vostru
+vouă
+vreo
+vreun
diff --git a/ckanext/multilingual/solr/schema.xml b/ckanext/multilingual/solr/schema.xml
new file mode 100644
index 0000000..8475187
--- /dev/null
+++ b/ckanext/multilingual/solr/schema.xml
@@ -0,0 +1,452 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="ckan" version="1.3">
+
+<types>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+ <fieldtype name="binary" class="solr.BinaryField"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0"/>
+
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <!-- in this example, we will only use synonyms at query time
+ <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+ -->
+ <!-- Case insensitive stop word removal.
+ add enablePositionIncrements=true in both the index and query
+ analyzers to leave a 'gap' for more accurate phrase queries.
+ -->
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+
+ <!-- A general unstemmed text field - good if one does not know the language of the field -->
+ <fieldType name="textgen" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- English -->
+
+ <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="stopwords.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- French -->
+
+ <fieldType name="text_fr" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="french_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.ElisionFilterFactory" articles="fr_elision.txt"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="French" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="french_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.ElisionFilterFactory" articles="fr_elision.txt"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="French" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- German -->
+
+ <fieldType name="text_de" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="german_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="German" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="german_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="German" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Spanish -->
+
+ <fieldType name="text_es" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="spanish_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Spanish" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="spanish_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Spanish" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Italian -->
+
+ <fieldType name="text_it" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="italian_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Italian" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="italian_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Italian" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Dutch -->
+
+ <fieldType name="text_nl" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="dutch_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Dutch" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="dutch_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Dutch" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Romanian -->
+
+ <fieldType name="text_ro" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="romanian_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Romanian" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="romanian_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Romanian" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Portuguese -->
+
+ <fieldType name="text_pt" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="portuguese_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Portuguese" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="portuguese_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" language="Portuguese" />
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- Polish -->
+
+ <fieldType name="text_pl" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="polish_stop.txt" enablePositionIncrements="true" />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="polish_stop.txt"
+ enablePositionIncrements="true"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+</types>
+
+<fields>
+ <field name="index_id" type="string" indexed="true" stored="true" required="true" />
+ <field name="id" type="string" indexed="true" stored="true" required="true" />
+ <field name="site_id" type="string" indexed="true" stored="true" required="true" />
+ <field name="title" type="text" indexed="true" stored="true" />
+ <field name="entity_type" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="state" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="name" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="revision_id" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="version" type="string" indexed="true" stored="true" />
+ <field name="url" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="ckan_url" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="download_url" type="string" indexed="true" stored="true" omitNorms="true" />
+ <field name="notes" type="text" indexed="true" stored="true"/>
+ <field name="author" type="textgen" indexed="true" stored="true" />
+ <field name="author_email" type="textgen" indexed="true" stored="true" />
+ <field name="maintainer" type="textgen" indexed="true" stored="true" />
+ <field name="maintainer_email" type="textgen" indexed="true" stored="true" />
+ <field name="license" type="string" indexed="true" stored="true" />
+ <field name="license_id" type="string" indexed="true" stored="true" />
+ <field name="ratings_count" type="int" indexed="true" stored="false" />
+ <field name="ratings_average" type="float" indexed="true" stored="false" />
+ <field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
+
+ <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>
+ <field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
+
+ <!-- catchall field, containing all other searchable text fields (implemented
+ via copyField further on in this schema -->
+ <field name="text" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="urls" type="text" indexed="true" stored="false" multiValued="true"/>
+
+ <field name="depends_on" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="dependency_of" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="derives_from" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="has_derivation" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="links_to" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="linked_from" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="child_of" type="text" indexed="true" stored="false" multiValued="true"/>
+ <field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
+
+ <field name="metadata_created" type="date" indexed="true" stored="true" multiValued="false"/>
+ <field name="metadata_modified" type="date" indexed="true" stored="true" multiValued="false"/>
+
+ <field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
+
+ <!-- Multilingual -->
+ <field name="text_en" type="text_en" indexed="true" stored="true"/>
+ <field name="title_en" type="text_en" indexed="true" stored="true"/>
+
+ <field name="text_fr" type="text_fr" indexed="true" stored="true"/>
+ <field name="title_fr" type="text_fr" indexed="true" stored="true"/>
+
+ <field name="text_de" type="text_de" indexed="true" stored="true"/>
+ <field name="title_de" type="text_de" indexed="true" stored="true"/>
+
+ <field name="text_es" type="text_es" indexed="true" stored="true"/>
+ <field name="title_es" type="text_es" indexed="true" stored="true"/>
+
+ <field name="text_it" type="text_it" indexed="true" stored="true"/>
+ <field name="title_it" type="text_it" indexed="true" stored="true"/>
+
+ <field name="text_nl" type="text_nl" indexed="true" stored="true"/>
+ <field name="title_nl" type="text_nl" indexed="true" stored="true"/>
+
+ <field name="text_ro" type="text_ro" indexed="true" stored="true"/>
+ <field name="title_ro" type="text_ro" indexed="true" stored="true"/>
+
+ <field name="text_pt" type="text_pt" indexed="true" stored="true"/>
+ <field name="title_pt" type="text_pt" indexed="true" stored="true"/>
+
+ <field name="text_pl" type="text_pl" indexed="true" stored="true"/>
+ <field name="title_pl" type="text_pl" indexed="true" stored="true"/>
+
+ <dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
+ <dynamicField name="*" type="string" indexed="true" stored="false"/>
+</fields>
+
+<uniqueKey>index_id</uniqueKey>
+<defaultSearchField>text</defaultSearchField>
+<solrQueryParser defaultOperator="AND"/>
+
+<copyField source="url" dest="urls"/>
+<copyField source="ckan_url" dest="urls"/>
+<copyField source="download_url" dest="urls"/>
+<copyField source="res_url" dest="urls"/>
+<copyField source="extras_*" dest="text"/>
+<copyField source="urls" dest="text"/>
+<copyField source="name" dest="text"/>
+<copyField source="title" dest="text"/>
+<copyField source="text" dest="text"/>
+<copyField source="license" dest="text"/>
+<copyField source="notes" dest="text"/>
+<copyField source="tags" dest="text"/>
+<copyField source="groups" dest="text"/>
+<copyField source="res_description" dest="text"/>
+<copyField source="maintainer" dest="text"/>
+<copyField source="author" dest="text"/>
+
+</schema>
diff --git a/ckanext/multilingual/solr/spanish_stop.txt b/ckanext/multilingual/solr/spanish_stop.txt
new file mode 100644
index 0000000..2db1476
--- /dev/null
+++ b/ckanext/multilingual/solr/spanish_stop.txt
@@ -0,0 +1,354 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ | - Encoding was converted to UTF-8.
+ | - This notice was added.
+
+ | A Spanish stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+
+ | The following is a ranked list (commonest to rarest) of stopwords
+ | deriving from a large sample of text.
+
+ | Extra words have been added at the end.
+
+de | from, of
+la | the, her
+que | who, that
+el | the
+en | in
+y | and
+a | to
+los | the, them
+del | de + el
+se | himself, from him etc
+las | the, them
+por | for, by, etc
+un | a
+para | for
+con | with
+no | no
+una | a
+su | his, her
+al | a + el
+ | es from SER
+lo | him
+como | how
+más | more
+pero | pero
+sus | su plural
+le | to him, her
+ya | already
+o | or
+ | fue from SER
+este | this
+ | ha from HABER
+sí | himself etc
+porque | because
+esta | this
+ | son from SER
+entre | between
+ | está from ESTAR
+cuando | when
+muy | very
+sin | without
+sobre | on
+ | ser from SER
+ | tiene from TENER
+también | also
+me | me
+hasta | until
+hay | there is/are
+donde | where
+ | han from HABER
+quien | whom, that
+ | están from ESTAR
+ | estado from ESTAR
+desde | from
+todo | all
+nos | us
+durante | during
+ | estados from ESTAR
+todos | all
+uno | a
+les | to them
+ni | nor
+contra | against
+otros | other
+ | fueron from SER
+ese | that
+eso | that
+ | había from HABER
+ante | before
+ellos | they
+e | and (variant of y)
+esto | this
+mí | me
+antes | before
+algunos | some
+qué | what?
+unos | a
+yo | I
+otro | other
+otras | other
+otra | other
+él | he
+tanto | so much, many
+esa | that
+estos | these
+mucho | much, many
+quienes | who
+nada | nothing
+muchos | many
+cual | who
+ | sea from SER
+poco | few
+ella | she
+estar | to be
+ | haber from HABER
+estas | these
+ | estaba from ESTAR
+ | estamos from ESTAR
+algunas | some
+algo | something
+nosotros | we
+
+ | other forms
+
+mi | me
+mis | mi plural
+tú | thou
+te | thee
+ti | thee
+tu | thy
+tus | tu plural
+ellas | they
+nosotras | we
+vosotros | you
+vosotras | you
+os | you
+mío | mine
+mía |
+míos |
+mías |
+tuyo | thine
+tuya |
+tuyos |
+tuyas |
+suyo | his, hers, theirs
+suya |
+suyos |
+suyas |
+nuestro | ours
+nuestra |
+nuestros |
+nuestras |
+vuestro | yours
+vuestra |
+vuestros |
+vuestras |
+esos | those
+esas | those
+
+ | forms of estar, to be (not including the infinitive):
+estoy
+estás
+está
+estamos
+estáis
+están
+esté
+estés
+estemos
+estéis
+estén
+estaré
+estarás
+estará
+estaremos
+estaréis
+estarán
+estaría
+estarías
+estaríamos
+estaríais
+estarían
+estaba
+estabas
+estábamos
+estabais
+estaban
+estuve
+estuviste
+estuvo
+estuvimos
+estuvisteis
+estuvieron
+estuviera
+estuvieras
+estuviéramos
+estuvierais
+estuvieran
+estuviese
+estuvieses
+estuviésemos
+estuvieseis
+estuviesen
+estando
+estado
+estada
+estados
+estadas
+estad
+
+ | forms of haber, to have (not including the infinitive):
+he
+has
+ha
+hemos
+habéis
+han
+haya
+hayas
+hayamos
+hayáis
+hayan
+habré
+habrás
+habrá
+habremos
+habréis
+habrán
+habría
+habrías
+habríamos
+habríais
+habrían
+había
+habías
+habíamos
+habíais
+habían
+hube
+hubiste
+hubo
+hubimos
+hubisteis
+hubieron
+hubiera
+hubieras
+hubiéramos
+hubierais
+hubieran
+hubiese
+hubieses
+hubiésemos
+hubieseis
+hubiesen
+habiendo
+habido
+habida
+habidos
+habidas
+
+ | forms of ser, to be (not including the infinitive):
+soy
+eres
+es
+somos
+sois
+son
+sea
+seas
+seamos
+seáis
+sean
+seré
+serás
+será
+seremos
+seréis
+serán
+sería
+serías
+seríamos
+seríais
+serían
+era
+eras
+éramos
+erais
+eran
+fui
+fuiste
+fue
+fuimos
+fuisteis
+fueron
+fuera
+fueras
+fuéramos
+fuerais
+fueran
+fuese
+fueses
+fuésemos
+fueseis
+fuesen
+siendo
+sido
+ | sed also means 'thirst'
+
+ | forms of tener, to have (not including the infinitive):
+tengo
+tienes
+tiene
+tenemos
+tenéis
+tienen
+tenga
+tengas
+tengamos
+tengáis
+tengan
+tendré
+tendrás
+tendrá
+tendremos
+tendréis
+tendrán
+tendría
+tendrías
+tendríamos
+tendríais
+tendrían
+tenía
+tenías
+teníamos
+teníais
+tenían
+tuve
+tuviste
+tuvo
+tuvimos
+tuvisteis
+tuvieron
+tuviera
+tuvieras
+tuviéramos
+tuvierais
+tuvieran
+tuviese
+tuvieses
+tuviésemos
+tuvieseis
+tuviesen
+teniendo
+tenido
+tenida
+tenidos
+tenidas
+tened
+
diff --git a/ckanext/multilingual/tests/test_multilingual_plugin.py b/ckanext/multilingual/tests/test_multilingual_plugin.py
new file mode 100644
index 0000000..9a09cc3
--- /dev/null
+++ b/ckanext/multilingual/tests/test_multilingual_plugin.py
@@ -0,0 +1,346 @@
+import ckan.plugins
+import ckanext.multilingual.plugin as mulilingual_plugin
+import ckan.lib.helpers
+import ckan.lib.create_test_data
+import ckan.logic.action.update
+import ckan.tests
+import ckan.tests.html_check
+import routes
+import paste.fixture
+import pylons.test
+import nose
+
+class TestDatasetTermTranslation(ckan.tests.html_check.HtmlCheckMethods):
+ '''Test the translation of datasets by the multilingual_dataset plugin.
+
+ '''
+ @classmethod
+ def setup(cls):
+ cls.app = paste.fixture.TestApp(pylons.test.pylonsapp)
+ ckan.plugins.load('multilingual_dataset')
+ ckan.plugins.load('multilingual_group')
+ ckan.plugins.load('multilingual_tag')
+ ckan.tests.setup_test_search_index()
+ ckan.lib.create_test_data.CreateTestData.create_translations_test_data()
+ # Add translation terms that match a couple of group names and package
+ # names. Group names and package names should _not_ get translated even
+ # if there are terms matching them, because they are used to form URLs.
+ for term in ('roger', 'david', 'annakarenina', 'warandpeace'):
+ for lang_code in ('en', 'de', 'fr'):
+ data_dict = {
+ 'term': term,
+ 'term_translation': 'this should not be rendered',
+ 'lang_code': lang_code,
+ }
+ context = {
+ 'model': ckan.model,
+ 'session': ckan.model.Session,
+ 'user': 'testsysadmin',
+ }
+ ckan.logic.action.update.term_translation_update(context,
+ data_dict)
+
+ @classmethod
+ def teardown(cls):
+ ckan.model.repo.rebuild_db()
+ ckan.lib.search.clear()
+
+ def test_dataset_read_translation(self):
+ '''Test the translation of dataset view pages by the
+ multilingual_dataset plugin.
+
+ '''
+ # Fetch the dataset view page for a number of different languages and
+ # test for the presence of translated and not translated terms.
+ offset = routes.url_for(controller='package', action='read',
+ id='annakarenina')
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ response = self.app.get(offset, status=200,
+ extra_environ={'CKAN_LANG': lang_code,
+ 'CKAN_CURRENT_URL': offset})
+ terms = ('A Novel By Tolstoy',
+ 'Index of the novel',
+ 'russian',
+ 'tolstoy',
+ "Dave's books",
+ "Roger's books",
+ 'Other (Open)',
+ 'romantic novel',
+ 'book',
+ '123',
+ '456',
+ '789',
+ 'plain text',
+ )
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
+ response.mustcontain('<a href="/tag/%s">' % tag_name)
+ for group_name in ('david', 'roger'):
+ response.mustcontain('<a href="/group/%s">' % group_name)
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_tag_read_translation(self):
+ '''Test the translation of tag view pages by the multilingual_tag
+ plugin.
+
+ '''
+ for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
+ offset = routes.url_for(controller='tag', action='read',
+ id=tag_name)
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ response = self.app.get(offset, status=200,
+ extra_environ={'CKAN_LANG': lang_code,
+ 'CKAN_CURRENT_URL': offset})
+ terms = ('A Novel By Tolstoy', tag_name, 'plain text', 'json')
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in (
+ ckan.lib.create_test_data.english_translations):
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[
+ term])
+ else:
+ response.mustcontain(term)
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_user_read_translation(self):
+ '''Test the translation of datasets on user view pages by the
+ multilingual_dataset plugin.
+
+ '''
+ for user_name in ('annafan',):
+ offset = routes.url_for(controller='user', action='read',
+ id=user_name)
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ response = self.app.get(offset, status=200,
+ extra_environ={'CKAN_LANG': lang_code,
+ 'CKAN_CURRENT_URL': offset})
+ terms = ('A Novel By Tolstoy', 'plain text', 'json')
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in (
+ ckan.lib.create_test_data.english_translations):
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[
+ term])
+ else:
+ response.mustcontain(term)
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_group_read_translation(self):
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ offset = '/%s/group/roger' % lang_code
+ response = self.app.get(offset, status=200)
+ terms = ('A Novel By Tolstoy',
+ 'Index of the novel',
+ 'russian',
+ 'tolstoy',
+ #"Dave's books",
+ "Roger's books",
+ #'Other (Open)',
+ #'romantic novel',
+ #'book',
+ '123',
+ '456',
+ '789',
+ 'plain text',
+ 'Roger likes these books.',
+ )
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
+ response.mustcontain('%s?tags=%s' % (offset, tag_name))
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_dataset_index_translation(self):
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ offset = '/%s/dataset' % lang_code
+ response = self.app.get(offset, status=200)
+ for term in ('Index of the novel', 'russian', 'tolstoy',
+ "Dave's books", "Roger's books", 'plain text'):
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
+ response.mustcontain('/%s/dataset?tags=%s' % (lang_code, tag_name))
+ for group_name in ('david', 'roger'):
+ response.mustcontain('/%s/dataset?groups=%s' % (lang_code, group_name))
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_group_index_translation(self):
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ offset = '/%s/group' % lang_code
+ response = self.app.get(offset, status=200)
+ terms = (
+ "Dave's books",
+ "Roger's books",
+ 'Roger likes these books.',
+ "These are books that David likes.",
+ )
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ for group_name in ('david', 'roger'):
+ response.mustcontain('/%s/group/%s' % (lang_code, group_name))
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+ def test_tag_index_translation(self):
+ for (lang_code, translations) in (
+ ('de', ckan.lib.create_test_data.german_translations),
+ ('fr', ckan.lib.create_test_data.french_translations),
+ ('en', ckan.lib.create_test_data.english_translations),
+ ('pl', {})):
+ offset = '/%s/tag' % lang_code
+ response = self.app.get(offset, status=200)
+ terms = (
+ "123",
+ "456",
+ '789',
+ "russian",
+ "tolstoy",
+ )
+ for term in terms:
+ if term in translations:
+ response.mustcontain(translations[term])
+ elif term in ckan.lib.create_test_data.english_translations:
+ response.mustcontain(
+ ckan.lib.create_test_data.english_translations[term])
+ else:
+ response.mustcontain(term)
+ response.mustcontain('/%s/tag/%s' % (lang_code, term))
+ nose.tools.assert_raises(IndexError, response.mustcontain,
+ 'this should not be rendered')
+
+class TestDatasetSearchIndex():
+
+ @classmethod
+ def setup_class(cls):
+ ckan.plugins.load('multilingual_dataset')
+ ckan.plugins.load('multilingual_group')
+
+ data_dicts = [
+ {'term': 'moo',
+ 'term_translation': 'french_moo',
+ 'lang_code': 'fr',
+ }, #
+ {'term': 'moo',
+ 'term_translation': 'this should not be rendered',
+ 'lang_code': 'fsdas',
+ },
+ {'term': 'an interesting note',
+ 'term_translation': 'french note',
+ 'lang_code': 'fr',
+ },
+ {'term': 'moon',
+ 'term_translation': 'french moon',
+ 'lang_code': 'fr',
+ },
+ {'term': 'boon',
+ 'term_translation': 'french boon',
+ 'lang_code': 'fr',
+ },
+ {'term': 'boon',
+ 'term_translation': 'italian boon',
+ 'lang_code': 'it',
+ },
+ {'term': 'david',
+ 'term_translation': 'french david',
+ 'lang_code': 'fr',
+ },
+ {'term': 'david',
+ 'term_translation': 'italian david',
+ 'lang_code': 'it',
+ },
+ ]
+
+ context = {
+ 'model': ckan.model,
+ 'session': ckan.model.Session,
+ 'user': 'testsysadmin',
+ 'ignore_auth': True,
+ }
+ for data_dict in data_dicts:
+ ckan.logic.action.update.term_translation_update(context,
+ data_dict)
+
+ def test_translate_terms(self):
+
+ sample_index_data = {
+ 'download_url': u'moo',
+ 'notes': u'an interesting note',
+ 'tags': [u'moon', 'boon'],
+ 'title': u'david',
+ }
+
+ result = mulilingual_plugin.MultilingualDataset().before_index(sample_index_data)
+
+ assert result == {'text_pl': '',
+ 'text_de': '',
+ 'text_ro': '',
+ 'title': u'david',
+ 'notes': u'an interesting note',
+ 'tags': [u'moon', 'boon'],
+ 'title_en': u'david',
+ 'download_url': u'moo',
+ 'text_it': u'italian boon',
+ 'text_es': '',
+ 'text_en': u'an interesting note moon boon moo',
+ 'text_nl': '',
+ 'title_it': u'italian david',
+ 'text_pt': '',
+ 'title_fr': u'french david',
+ 'text_fr': u'french note french boon french_moo french moon'}, result
diff --git a/doc/_themes/sphinx-theme-okfn b/doc/_themes/sphinx-theme-okfn
index 5c2ef73..ef101a1 160000
--- a/doc/_themes/sphinx-theme-okfn
+++ b/doc/_themes/sphinx-theme-okfn
@@ -1 +1 @@
-Subproject commit 5c2ef7385a9e1012d0c328722c8cb235f18391a1
+Subproject commit ef101a18d6de959207361e233b89efd3be24e66f
diff --git a/doc/api.rst b/doc/api.rst
index 18e6823..98d84dc 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -1,4 +1,5 @@
.. index:: API
+.. _api:
========
CKAN API
diff --git a/doc/apiv3.rst b/doc/apiv3.rst
index 22624f6..2ee71ef 100644
--- a/doc/apiv3.rst
+++ b/doc/apiv3.rst
@@ -73,10 +73,14 @@ revision_show id
group_show id
tag_show id
user_show id
+term_translation_show "**terms**" A list of strings, the terms that you want to search for translations of, e.g. "russian", "romantic novel". "**lang_codes**" A list of strings, language codes for the languages that you want to search for translations to, e.g. "en", "de". Optional, if no lang_codes are given translations to all languages will be returned.
package_show_rest id
group_show_rest id
tag_show_rest id
vocabulary_show id
+task_status_show id
+task_status_show entity_id, task_type, key
+resource_status_show id
package_autocomplete q
tag_autocomplete q, fields, offset, limit, vocabulary_id
format_autocomplete q, limit
@@ -122,6 +126,8 @@ group_update_rest (group keys)
user_role_update user OR authorization_group, domain_object, roles
user_role_bulk_update user_roles, domain_object
vocabulary_update (vocabulary keys)
+term_translation_update "**term**" The term that you want to create (or update) a translation for, e.g. "russian", "romantic novel". "**term_translation**" the translation of the term, e.g. "Russisch", "Liebesroman". "**lang_code**" the language code for the translation, e.g. "fr", "de".
+term_translation_update_many "**data**" A list of dictionaries with keys matching the parameter keys for term_translation_update
====================================== ===========================
delete.py:
@@ -205,6 +211,7 @@ key example value notes
======================== ====================================== =============
id "b10871ea-b4ae-4e2e-bec9-a8d8ff357754" (Read-only)
name "country-uk" (Read-only) Add/remove tags from a package or group using update_package or update_group
+display_name "country-uk" (Read-only) display_name is the name of the tag that is displayed to user (as opposed to name which is used to identify the tag, e.g. in URLs). display_name is is usually the same as name but may be different, for example display_names may be translated by the ckanext-multilingual extension.
state "active" (Read-only) Add/remove tags from a package or group using update_package or update_group
revision_timestamp "2009-08-08T12:46:40.920443" (Read-only)
vocabulary_id "Genre" (Read-only) Vocabulary name or id. Optional.
@@ -230,6 +237,16 @@ name "Genre"
tags [{"name":"government-spending"}, {"name": "climate"}] List of tags belonging to this vocabulary.
======================== ===================================================== =============
+Term Translation:
+
+================ ========================= ==================================
+key example value notes
+================ ========================= ==================================
+term "russian" The term that is being translated.
+term_translation "Russisch" The translation of the term.
+lang_code "de" The language of the translation, a language code string.
+================ ========================= ==================================
+
Parameters
==========
diff --git a/doc/datastore.rst b/doc/datastore.rst
index 395de84..15fcd8f 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -30,30 +30,21 @@ the spreadsheet data is stored in the DataStore one would be able to access
individual spreadsheet rows via a simple web-api as well as being able to make
queries over the spreadsheet contents.
-Using the DataStore Data API
-============================
+The DataStore Data API
+======================
The DataStore's Data API, which derives from the underlying ElasticSearch
data-table, is RESTful and JSON-based with extensive query capabilities.
-Each resource in a CKAN instance has an associated DataStore 'database'. This
-database will be accessible via a web interface at::
+Each resource in a CKAN instance has an associated DataStore 'table'. This
+table will be accessible via a web interface at::
/api/data/{resource-id}
This interface to this data is *exactly* the same as that provided by
ElasticSearch to documents of a specific type in one of its indices.
-So, for example, to see the fields in this database do::
-
- /api/data/{resource-id}/_mapping
-
-To do simple search do::
-
- /api/data/{resource-id}/_search?q=abc
-
-For more on searching see: http://www.elasticsearch.org/guide/reference/api/search/uri-request.html
-
+For a detailed tutorial on using this API see :doc:`using-data-api`.
Installation and Configuration
==============================
diff --git a/doc/i18n.rst b/doc/i18n.rst
index 07c2c50..7361525 100644
--- a/doc/i18n.rst
+++ b/doc/i18n.rst
@@ -1,3 +1,5 @@
+.. _i18n:
+
=====================
Internationalize CKAN
=====================
diff --git a/doc/index.rst b/doc/index.rst
index 4fe9aaf..67e1153 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -39,6 +39,7 @@ Customizing and Extending
datastore
background-tasks
geospatial
+ multilingual
Publishing Datasets
===================
@@ -59,6 +60,8 @@ The CKAN API
:maxdepth: 3
api
+ api-tutorial
+ using-data-api
General Administration
======================
diff --git a/doc/multilingual.rst b/doc/multilingual.rst
new file mode 100644
index 0000000..a465b04
--- /dev/null
+++ b/doc/multilingual.rst
@@ -0,0 +1,39 @@
+=====================================
+Translating Datasets, Groups and Tags
+=====================================
+
+For translating CKAN's web interface see :ref:`i18n`. In addition to user interface internationalization, a CKAN administrator can also enter translations into CKAN's database for terms that may appear in the contents of datasets, groups or tags created by users. When a user is viewing the CKAN site, if the translation terms database contains a translation in the user's language for the name or description of a dataset or resource, the name of a tag or group, etc. then the translated term will be shown to the user in place of the original.
+
+Setup and Configuration
+-----------------------
+
+By default term translations are disabled. To enable them, you have to specify the multilingual plugins using the ``ckan.plugins`` setting in your CKAN configuration file, for example:
+
+::
+
+ # List the names of CKAN extensions to activate.
+ ckan.plugins = multilingual_dataset multilingual_group multilingual_tag
+
+Of course, you won't see any terms getting translated until you load some term translations into the database. You can do this using the ``term_translation_update`` and ``term_translation_update_many`` actions of the CKAN API, See :ref:`api` for more details.
+
+Loading Test Translations
+-------------------------
+
+If you want to quickly test the term translation feature without having to provide your own translations, you can load CKAN's test translations into the database by running this command from your shell:
+
+::
+
+ paster --plugin=ckan create-test-data translations
+
+See :ref:`paster` for more details.
+
+Testing The Multilingual Extension
+----------------------------------
+
+If you have a source installation of CKAN you can test the multilingual extension by running the tests located in ``ckanext/multilingual/tests``. You must first install the packages needed for running CKAN tests into your virtual environment, and then run this command from your shell:
+
+::
+
+ nosetests --ckan ckanext/multilingual/tests
+
+See :ref:`basic-tests` for more information.
diff --git a/doc/paster.rst b/doc/paster.rst
index 19d197b..bf94e9f 100644
--- a/doc/paster.rst
+++ b/doc/paster.rst
@@ -1,3 +1,5 @@
+.. _paster:
+
===============================
Common CKAN Administrator Tasks
===============================
diff --git a/doc/post-installation.rst b/doc/post-installation.rst
index 5e39124..5ef039f 100644
--- a/doc/post-installation.rst
+++ b/doc/post-installation.rst
@@ -45,7 +45,13 @@ It can be handy to have some test data to start with. You can get test data like
paster --plugin=ckan create-test-data --config=/etc/ckan/std/std.ini
You now have a CKAN instance that you can log in to, with some test data to check everything
-works.
+works.
+
+You can also create various specialised test data collections for testing specific features of CKAN. For example, ``paster --plugin=ckan create-test-data translations`` creates some test data with some translations for testing the ckanext-multilingual extension. For more information, see:
+
+::
+
+ paster --plugin=ckan create-test-data --help
.. _deployment-notes:
diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst
new file mode 100644
index 0000000..3c6a6fe
--- /dev/null
+++ b/doc/using-data-api.rst
@@ -0,0 +1,402 @@
+==================
+Using the Data API
+==================
+
+Introduction
+============
+
+The Data API builds directly on ElasticSearch, with a resource API endpoint
+being equivalent to a single index 'type' in ElasticSearch (we tend to refer to
+it as a 'table'). This means you can often directly re-use `ElasticSearch
+client libraries`_ when connecting to the API endpoint.
+
+Furthermore, it means that what is presented below is essentially a tutorial in the ElasticSearch API.
+
+.. _ElasticSearch client libraries: http://www.elasticsearch.org/guide/appendix/clients.html
+
+Quickstart
+==========
+
+``endpoint`` refers to the data API endpoint (or ElasticSearch index / table).
+
+Key urls:
+
+* Query: ``{endpoint}/_search`` (in ElasticSearch < 0.19 this will return an
+ error if visited without a query parameter)
+
+ * Query example: ``{endpoint}/_search?size=5&pretty=true``
+
+* Schema (Mapping): ``{endpoint}/_mapping``
+
+Examples
+--------
+
+cURL (or Browser)
+~~~~~~~~~~~~~~~~~
+
+The following examples utilize the <a href="http://curl.haxx.se/">cURL</a>
+command line utility. If you prefer, you you can just open the relevant urls in
+your browser::
+
+ // added pretty=true to get the json results pretty printed
+ curl {endpoint}/_search?q=title:jones&size=5&pretty=true</pre>
+
+Javascript
+~~~~~~~~~~~
+
+A simple ajax (JSONP) request to the data API using jQuery::
+
+ var data = {
+ size: 5 // get 5 results
+ q: 'title:jones' // query on the title field for 'jones'
+ };
+ $.ajax({
+ url: {endpoint}/_search,
+ dataType: 'jsonp',
+ success: function(data) {
+ alert('Total results found: ' + data.hits.total)
+ }
+ });
+
+
+Querying
+========
+
+Basic Queries Using Only the Query String
+-----------------------------------------
+
+Basic queries can be done using only query string parameters in the URL. For
+example, the following searches for text 'hello' in any field in any document
+and returns at most 5 results::
+
+ {endpoint}/_search?q=hello&size=5
+
+Basic queries like this have the advantage that they only involve accessing a
+URL and thus, for example, can be performed just using any web browser.
+However, this method is limited and does not give you access to most of the
+more powerful query features.
+
+Basic queries use the `q` query string parameter which supports the `Lucene
+query parser syntax`_ and hence filters on specific fields (e.g. `fieldname:value`), wildcards (e.g. `abc*`) and more.
+
+.. _Lucene query parser syntax: http://lucene.apache.org/core/old_versioned_docs/versions/3_0_0/queryparsersyntax.html
+
+There are a variety of other options (e.g. size, from etc) that you can also
+specify to customize the query and its results. Full details can be found in
+the `ElasticSearch URI request docs`_.
+
+.. _ElasticSearch URI request docs: http://www.elasticsearch.org/guide/reference/api/search/uri-request.html
+
+Full Query API
+--------------
+
+More powerful and complex queries, including those that involve faceting and
+statistical operations, should use the full ElasticSearch query language and API.
+
+In the query language queries are written as a JSON structure and is then sent
+to the query endpoint (details of the query langague below). There are two
+options for how a query is sent to the search endpoint:
+
+1. Either as the value of a source query parameter e.g.::
+
+ {endpoint}/_search?source={Query-as-JSON}
+
+2. Or in the request body, e.g.::
+
+ curl -XGET {endpoint}/_search -d 'Query-as-JSON'
+
+ For example::
+
+ curl -XGET {endpoint}/_search -d '{
+ "query" : {
+ "term" : { "user": "kimchy" }
+ }
+ }'
+
+
+Query Language
+==============
+
+Queries are JSON objects with the following structure (each of the main
+sections has more detail below)::
+
+ {
+ size: # number of results to return (defaults to 10)
+ from: # offset into results (defaults to 0)
+ fields: # list of document fields that should be returned - http://elasticsearch.org/guide/reference/api/search/fields.html
+ sort: # define sort order - see http://elasticsearch.org/guide/reference/api/search/sort.html
+
+ query: {
+ # "query" object following the Query DSL: http://elasticsearch.org/guide/reference/query-dsl/
+ # details below
+ },
+
+ facets: {
+ # facets specifications
+ # Facets provide summary information about a particular field or fields in the data
+ }
+
+ # special case for situations where you want to apply filter/query to results but *not* to facets
+ filter: {
+ # filter objects
+ # a filter is a simple "filter" (query) on a specific field.
+ # Simple means e.g. checking against a specific value or range of values
+ },
+ }
+
+Query results look like::
+
+ {
+ # some info about the query (which shards it used, how long it took etc)
+ ...
+ # the results
+ hits: {
+ total: # total number of matching documents
+ hits: [
+ # list of "hits" returned
+ {
+ _id: # id of document
+ score: # the search index score
+ _source: {
+ # document 'source' (i.e. the original JSON document you sent to the index
+ }
+ }
+ ]
+ }
+ # facets if these were requested
+ facets: {
+ ...
+ }
+ }
+
+Query DSL: Overview
+-------------------
+
+Query objects are built up of sub-components. These sub-components are either
+basic or compound. Compound sub-components may contains other sub-components
+while basic may not. Example::
+
+ {
+ "query": {
+ # compound component
+ "bool": {
+ # compound component
+ "must": {
+ # basic component
+ "term": {
+ "user": "jones"
+ }
+ }
+ # compound component
+ "must_not": {
+ # basic component
+ "range" : {
+ "age" : {
+ "from" : 10,
+ "to" : 20
+ }
+ }
+ }
+ }
+ }
+ }
+
+In addition, and somewhat confusingly, ElasticSearch distinguishes between
+sub-components that are "queries" and those that are "filters". Filters, are
+really special kind of queries that are: mostly basic (though boolean
+compounding is alllowed); limited to one field or operation and which, as such,
+are especially performant.
+
+Examples, of filters are (full list on RHS at the bottom of the query-dsl_ page):
+
+ * term: filter on a value for a field
+ * range: filter for a field having a range of values (>=, <= etc)
+ * geo_bbox: geo bounding box
+ * geo_distance: geo distance
+
+.. _query-dsl: http://elasticsearch.org/guide/reference/query-dsl/
+
+Rather than attempting to set out all the constraints and options of the
+query-dsl we now offer a variety of examples.
+
+Examples
+--------
+
+Match all / Find Everything
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ {
+ "query": {
+ "match_all": {}
+ }
+ }
+
+Classic Search-Box Style Full-Text Query
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This will perform a full-text style query across all fields. The query string
+supports the `Lucene query parser syntax`_ and hence filters on specific fields
+(e.g. `fieldname:value`), wildcards (e.g. `abc*`) as well as a variety of
+options. For full details see the query-string_ documentation.
+
+::
+
+ {
+ "query": {
+ "query_string": {
+ "query": {query string}
+ }
+ }
+ }
+
+.. _query-string: http://elasticsearch.org/guide/reference/query-dsl/query-string-query.html
+
+Filter on One Field
+~~~~~~~~~~~~~~~~~~~
+
+::
+
+ {
+ "query": {
+ "term": {
+ {field-name}: {value}
+ }
+ }
+ }
+
+High performance equivalent using filters::
+
+ {
+ "query": {
+ "constant_score": {
+ "filter": {
+ "term": {
+ # note that value should be *lower-cased*
+ {field-name}: {value}
+ }
+ }
+ }
+ }
+
+Find all documents with value in a range
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This can be used both for text ranges (e.g. A to Z), numeric ranges (10-20) and
+for dates (ElasticSearch will converts dates to ISO 8601 format so you can
+search as 1900-01-01 to 1920-02-03).
+
+::
+
+ {
+ "query": {
+ "constant_score": {
+ "filter": {
+ "range": {
+ {field-name}: {
+ "from": {lower-value}
+ "to": {upper-value}
+ }
+ }
+ }
+ }
+ }
+ }
+
+For more details see `range filters`_.
+
+.. _range filters: http://elasticsearch.org/guide/reference/query-dsl/range-filter.html
+
+Full-Text Query plus Filter on a Field
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ {
+ "query": {
+ "query_string": {
+ "query": {query string}
+ },
+ "term": {
+ {field}: {value}
+ }
+ }
+ }
+
+
+Filter on two fields
+~~~~~~~~~~~~~~~~~~~~
+
+Note that you cannot, unfortunately, have a simple and query by adding two
+filters inside the query element. Instead you need an 'and' clause in a filter
+(which in turn requires nesting in 'filtered'). You could also achieve the same
+result here using a `bool query`_.
+
+.. _bool query: http://elasticsearch.org/guide/reference/query-dsl/bool-query.html
+
+::
+
+ {
+ "query": {
+ "filtered": {
+ "query": {
+ "match_all": {}
+ },
+ "filter": {
+ "and": [
+ {
+ "range" : {
+ "b" : {
+ "from" : 4,
+ "to" : "8"
+ }
+ },
+ },
+ {
+ "term": {
+ "a": "john"
+ }
+ }
+ ]
+ }
+ }
+ }
+ }
+
+Facets
+------
+
+Facets provide a way to get summary information about then data in an
+elasticsearch table, for example counts of distinct values.
+
+TODO: complete
+
+
+Schema Mapping
+==============
+
+As the ElasticSearch documentation states:
+
+ Mapping is the process of defining how a document should be mapped to the
+ Search Engine, including its searchable characteristics such as which fields
+ are searchable and if/how they are tokenized. In ElasticSearch, an index may
+ store documents of different “mapping types”. ElasticSearch allows one to
+ associate multiple mapping definitions for each mapping type.
+
+ Explicit mapping is defined on an index/type level. By default, there isn't a
+ need to define an explicit mapping, since one is automatically created and
+ registered when a new type or new field is introduced (with no performance
+ overhead) and have sensible defaults. Only when the defaults need to be
+ overridden must a mapping definition be provided.
+
+Relevant docs: http://elasticsearch.org/guide/reference/mapping/.
+
+
+JSONP support
+=============
+
+JSONP support is available on any request via a simple callback query string parameter::
+
+ ?callback=my_callback_name
+
diff --git a/setup.py b/setup.py
index ee1f06d..33f6d6b 100644
--- a/setup.py
+++ b/setup.py
@@ -91,6 +91,11 @@
[ckan.plugins]
synchronous_search = ckan.lib.search:SynchronousSearchPlugin
stats=ckanext.stats.plugin:StatsPlugin
+ publisher_form=ckanext.publisher_form.forms:PublisherForm
+ publisher_dataset_form=ckanext.publisher_form.forms:PublisherDatasetForm
+ multilingual_dataset=ckanext.multilingual.plugin:MultilingualDataset
+ multilingual_group=ckanext.multilingual.plugin:MultilingualGroup
+ multilingual_tag=ckanext.multilingual.plugin:MultilingualTag
organizations=ckanext.organizations.forms:OrganizationForm
organizations_dataset=ckanext.organizations.forms:OrganizationDatasetForm
test_tag_vocab_plugin=ckanext.test_tag_vocab_plugin:MockVocabTagsPlugin
================================================================
Commit: 6304264f02b3f26d9001aa447353e6c2799f53f0
https://github.com/okfn/ckan/commit/6304264f02b3f26d9001aa447353e6c2799f53f0
Author: Ross Jones <rossdjones at gmail.com>
Date: 2012-04-23 (Mon, 23 Apr 2012)
Changed paths:
M ckan/plugins/toolkit.py
Log Message:
-----------
Merge branch 'master' of https://github.com/okfn/ckan
diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py
index 3eb8983..671449e 100644
--- a/ckan/plugins/toolkit.py
+++ b/ckan/plugins/toolkit.py
@@ -40,7 +40,7 @@ class _Toolkit(object):
'literal', # stop tags in a string being escaped
'get_action', # get logic action function
'check_access', # check logic function authorisation
- 'ActionNotFound', # action not found exception (ckan.logic.NotFound)
+ 'ObjectNotFound', # action not found exception (ckan.logic.NotFound)
'NotAuthorized', # action not authorized exception
'ValidationError', # model update validation error
'CkanCommand', # class for providing cli interfaces
@@ -85,7 +85,7 @@ def _initialize(self):
t['get_action'] = logic.get_action
t['check_access'] = logic.check_access
- t['ActionNotFound'] = logic.NotFound ## Name change intentional
+ t['ObjectNotFound'] = logic.NotFound ## Name change intentional
t['NotAuthorized'] = logic.NotAuthorized
t['ValidationError'] = logic.ValidationError
================================================================
Commit: b715eb44f3d9889d54048575cfacc3295669aa45
https://github.com/okfn/ckan/commit/b715eb44f3d9889d54048575cfacc3295669aa45
Author: Ross Jones <rossdjones at gmail.com>
Date: 2012-04-23 (Mon, 23 Apr 2012)
Changed paths:
M requires/lucid_missing.txt
Log Message:
-----------
Merge branch 'master' of https://github.com/okfn/ckan
diff --git a/requires/lucid_missing.txt b/requires/lucid_missing.txt
index 8b5789e..dccb362 100644
--- a/requires/lucid_missing.txt
+++ b/requires/lucid_missing.txt
@@ -20,3 +20,4 @@ ofs==0.4.1
apachemiddleware==0.1.1
# markupsafe is required by webhelpers==1.2 required by formalchemy with SQLAlchemy 0.6
markupsafe==0.9.2
+celery==2.5.3
================================================================
Commit: 969c19dc0ff388a742bdc2fc3d56256411d693d7
https://github.com/okfn/ckan/commit/969c19dc0ff388a742bdc2fc3d56256411d693d7
Author: Ross Jones <rossdjones at gmail.com>
Date: 2012-04-24 (Tue, 24 Apr 2012)
Changed paths:
M ckan/lib/cli.py
M ckan/logic/converters.py
M ckan/plugins/toolkit.py
M doc/api.rst
A doc/contrib.rst
M doc/index.rst
M doc/using-data-api.rst
M setup.py
Log Message:
-----------
Merge branch 'master' of https://github.com/okfn/ckan
diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
index 6798cc4..fd52cc6 100644
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -879,3 +879,96 @@ def clean(self, user_ratings=True):
rating.purge()
model.repo.commit_and_remove()
+
+class PluginInfo(CkanCommand):
+ ''' Provide info on installed plugins.
+ '''
+
+ summary = __doc__.split('\n')[0]
+ usage = __doc__
+ max_args = 0
+ min_args = 0
+
+ def command(self):
+ self.get_info()
+
+ def get_info(self):
+ ''' print info about current plugins from the .ini file'''
+ import ckan.plugins as p
+ self._load_config()
+ interfaces = {}
+ plugins = {}
+ for name in dir(p):
+ item = getattr(p, name)
+ try:
+ if issubclass(item, p.Interface):
+ interfaces[item] = {'class' : item}
+ except TypeError:
+ pass
+
+ for interface in interfaces:
+ for plugin in p.PluginImplementations(interface):
+ name = plugin.name
+ if name not in plugins:
+ plugins[name] = {'doc' : plugin.__doc__,
+ 'class' : plugin,
+ 'implements' : []}
+ plugins[name]['implements'].append(interface.__name__)
+
+ for plugin in plugins:
+ p = plugins[plugin]
+ print plugin + ':'
+ print '-' * (len(plugin) + 1)
+ if p['doc']:
+ print p['doc']
+ print 'Implements:'
+ for i in p['implements']:
+ extra = None
+ if i == 'ITemplateHelpers':
+ extra = self.template_helpers(p['class'])
+ if i == 'IActions':
+ extra = self.actions(p['class'])
+ print ' %s' % i
+ if extra:
+ print extra
+ print
+
+
+ def actions(self, cls):
+ ''' Return readable action function info. '''
+ actions = cls.get_actions()
+ return self.function_info(actions)
+
+ def template_helpers(self, cls):
+ ''' Return readable helper function info. '''
+ helpers = cls.get_helpers()
+ return self.function_info(helpers)
+
+ def function_info(self, functions):
+ ''' Take a dict of functions and output readable info '''
+ import inspect
+ output = []
+ for function_name in functions:
+ fn = functions[function_name]
+ args_info = inspect.getargspec(fn)
+ params = args_info.args
+ num_params = len(params)
+ if args_info.varargs:
+ params.append('*' + args_info.varargs)
+ if args_info.keywords:
+ params.append('**' + args_info.keywords)
+ if args_info.defaults:
+ offset = num_params - len(args_info.defaults)
+ for i, v in enumerate(args_info.defaults):
+ params[i + offset] = params[i + offset] + '=' + repr(v)
+ # is this a classmethod if so remove the first parameter
+ if inspect.ismethod(fn) and inspect.isclass(fn.__self__):
+ params = params[1:]
+ params = ', '.join(params)
+ output.append(' %s(%s)' % (function_name, params))
+ # doc string
+ if fn.__doc__:
+ bits = fn.__doc__.split('\n')
+ for bit in bits:
+ output.append(' %s' % bit)
+ return ('\n').join(output)
diff --git a/ckan/logic/converters.py b/ckan/logic/converters.py
index 5de75a7..6b927ae 100644
--- a/ckan/logic/converters.py
+++ b/ckan/logic/converters.py
@@ -60,13 +60,11 @@ def callable(key, data, errors, context):
context['vocabulary'] = v
for tag in new_tags:
- tag_length_validator(tag, context)
- tag_name_validator(tag, context)
tag_in_vocabulary_validator(tag, context)
for num, tag in enumerate(new_tags):
- data[('tags', num+n, 'name')] = tag
- data[('tags', num+n, 'vocabulary_id')] = v.id
+ data[('tags', num + n, 'name')] = tag
+ data[('tags', num + n, 'vocabulary_id')] = v.id
return callable
def convert_from_tags(vocab):
diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py
index 671449e..fa1f2c3 100644
--- a/ckan/plugins/toolkit.py
+++ b/ckan/plugins/toolkit.py
@@ -183,6 +183,8 @@ def __getattr__(self, name):
if name in self._toolkit:
return self._toolkit[name]
else:
+ if name == '__bases__':
+ return self.__class__.__bases__
raise Exception('`%s` not found in plugins toolkit' % name)
toolkit = _Toolkit()
diff --git a/doc/api.rst b/doc/api.rst
index 98d84dc..51deb4a 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -91,6 +91,8 @@ key is not authorized for the operation, or the header is somehow malformed,
then the requested operation will not be carried out and the CKAN API will
respond with status code 403.
+.. _get-api-key:
+
Obtaining an API key
--------------------
diff --git a/doc/contrib.rst b/doc/contrib.rst
new file mode 100644
index 0000000..87e1967
--- /dev/null
+++ b/doc/contrib.rst
@@ -0,0 +1,73 @@
+=================
+Contrib and Tools
+=================
+
+This is a a place for code, scripts and applications that extend CKAN in some
+way (but which aren't extensions).
+
+.. note:: Have something to add to this page? Please either email ``info [at] ckan [dot]
+ org`` or just edit this `page in the CKAN repo`_.
+
+.. _page in the CKAN repo: https://github.com/okfn/ckan/blob/master/doc/contrib.rst
+
+Google Docs Integration
+=======================
+
+Integration of Google docs spreadsheets with CKAN. Google app script script which supports:
+
+* Publish a Google docs spreadsheet as a dataset on CKAN
+* Push and pull metadata from a CKAN instance to a Google docs spreadsheet
+
+Install: via the google docs Script Gallery in the tools menu (or use the source).
+
+Repo: https://github.com/okfn/ckan-google-docs
+
+Google Refine Extension for CKAN
+================================
+
+This extension allows data of Google Refine projects to be uploaded to CKAN
+Storage and connected to a dataset on a running CKAN instance (for example
+http://thedatahub.org).
+
+* http://lab.linkeddata.deri.ie/2011/grefine-ckan/
+* Intro: http://ckan.org/2011/07/05/google-refine-extension-for-ckan/
+* Author: Fadi Maali (DERI)
+
+Repo: https://github.com/fadmaa/grefine-ckan-storage-extension
+
+Embeddable Dataset Count Widget
+===============================
+
+Simple group count widget in pure javascript for embedding in other websites.
+
+http://okfnlabs.org/ckanjs/widgets/count/
+
+Embeddable Dataset Listing Widget
+=================================
+
+http://okfnlabs.org/ckanjs/widgets/search/
+
+SPARQL Endpoint Status Checker
+==============================
+
+Status of SPARQL endpoints (in the DataHub).
+
+http://labs.mondeca.com/sparqlEndpointsStatus/index.html
+
+* Author: Mondeca
+
+PublicData.eu Data map
+======================
+
+http://publicdata.eu/map
+
+VoID metadata generator
+=======================
+
+Useful to automatically correct/maintain/enrich dataset metadata
+
+* voidGen : http://www.hpi.uni-potsdam.de/naumann/projects/btc
+* ve2 : http://lab.linkeddata.deri.ie/ve2/
+* (POC by Richard) : https://github.com/cygri/make-void
+* (POC by Pierre-Yves) : http://labs.mondeca.com/vocabuse/
+
diff --git a/doc/index.rst b/doc/index.rst
index 67e1153..b1c71f0 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -93,8 +93,9 @@ Other material
==============
.. toctree::
- :maxdepth: 1
+ :maxdepth: 2
+ contrib.rst
CHANGELOG.rst
Indices and tables
diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst
index 3c6a6fe..09caf21 100644
--- a/doc/using-data-api.rst
+++ b/doc/using-data-api.rst
@@ -17,16 +17,16 @@ Furthermore, it means that what is presented below is essentially a tutorial in
Quickstart
==========
-``endpoint`` refers to the data API endpoint (or ElasticSearch index / table).
+``{{endpoint}}`` refers to the data API endpoint (or ElasticSearch index / table).
Key urls:
-* Query: ``{endpoint}/_search`` (in ElasticSearch < 0.19 this will return an
+* Query: ``{{endpoint}}/_search`` (in ElasticSearch < 0.19 this will return an
error if visited without a query parameter)
- * Query example: ``{endpoint}/_search?size=5&pretty=true``
+ * Query example: ``{{endpoint}}/_search?size=5&pretty=true``
-* Schema (Mapping): ``{endpoint}/_mapping``
+* Schema (Mapping): ``{{endpoint}}/_mapping``
Examples
--------
@@ -34,15 +34,26 @@ Examples
cURL (or Browser)
~~~~~~~~~~~~~~~~~
-The following examples utilize the <a href="http://curl.haxx.se/">cURL</a>
-command line utility. If you prefer, you you can just open the relevant urls in
-your browser::
+The following examples utilize the cURL_ command line utility. If you prefer,
+you you can just open the relevant urls in your browser::
+ // query for documents / rows with title field containing 'jones'
// added pretty=true to get the json results pretty printed
- curl {endpoint}/_search?q=title:jones&size=5&pretty=true</pre>
+ curl {{endpoint}}/_search?q=title:jones&size=5&pretty=true
+
+Adding some data (requires an :ref:`API Key <get-api-key>`)::
+
+ // requires an API key
+ // Data (argument to -d) should be a JSON document
+ curl -X POST -H "Authorization: {{YOUR-API-KEY}}" {{endpoint}} -d '{
+ "title": "jones",
+ "amount": 5.7
+ }'
+
+.. _cURL: http://curl.haxx.se/
Javascript
-~~~~~~~~~~~
+~~~~~~~~~~
A simple ajax (JSONP) request to the data API using jQuery::
@@ -51,13 +62,71 @@ A simple ajax (JSONP) request to the data API using jQuery::
q: 'title:jones' // query on the title field for 'jones'
};
$.ajax({
- url: {endpoint}/_search,
+ url: {{endpoint}}/_search,
dataType: 'jsonp',
success: function(data) {
alert('Total results found: ' + data.hits.total)
}
});
+The Data API supports CORs so you can also write to it (this requires the json2_ library for ``JSON.stringify``)::
+
+ var data = {
+ title: 'jones',
+ amount: 5.7
+ };
+ $.ajax({
+ url: {{endpoint}},
+ type: 'POST',
+ data: JSON.stringify(data),
+ success: function(data) {
+ alert('Uploaded ok')
+ }
+ });
+
+.. _json2: https://github.com/douglascrockford/JSON-js/blob/master/json2.js
+
+Python
+~~~~~~
+
+.. note:: You can also use the `DataStore Python client library`_.
+
+.. _DataStore Python client library: http://github.com/okfn/datastore-client
+
+::
+
+ import urllib2
+ import json
+
+ # =================================
+ # Store data in the DataStore table
+
+ url = '{{endpoint}}'
+ data = {
+ 'title': 'jones',
+ 'amount': 5.7
+ }
+ # have to send the data as JSON
+ data = json.dumps(data)
+ # need to add your API key (and have authorization to write to this endpoint)
+ headers = {'Authorization': 'YOUR-API-KEY'}
+
+ req = urllib2.Request(url, data, headers)
+ out = urllib2.urlopen(req)
+ print out.read()
+
+ # =========================
+ # Query the DataStore table
+
+ url = '{{endpoint}}/_search?q=title:jones&size=5'
+ req = urllib2.Request(url)
+ out = urllib2.urlopen(req)
+ data = out.read()
+ print data
+ # returned data is JSON
+ data = json.loads(data)
+ # total number of results
+ print data['hits']['total']
Querying
========
@@ -69,7 +138,7 @@ Basic queries can be done using only query string parameters in the URL. For
example, the following searches for text 'hello' in any field in any document
and returns at most 5 results::
- {endpoint}/_search?q=hello&size=5
+ {{endpoint}}/_search?q=hello&size=5
Basic queries like this have the advantage that they only involve accessing a
URL and thus, for example, can be performed just using any web browser.
@@ -99,15 +168,15 @@ options for how a query is sent to the search endpoint:
1. Either as the value of a source query parameter e.g.::
- {endpoint}/_search?source={Query-as-JSON}
+ {{endpoint}}/_search?source={Query-as-JSON}
2. Or in the request body, e.g.::
- curl -XGET {endpoint}/_search -d 'Query-as-JSON'
+ curl -XGET {{endpoint}}/_search -d 'Query-as-JSON'
For example::
- curl -XGET {endpoint}/_search -d '{
+ curl -XGET {{endpoint}}/_search -d '{
"query" : {
"term" : { "user": "kimchy" }
}
@@ -364,13 +433,54 @@ result here using a `bool query`_.
}
}
+
Facets
------
Facets provide a way to get summary information about then data in an
elasticsearch table, for example counts of distinct values.
-TODO: complete
+ElasticSearch (and hence the Data API) provides rich faceting capabilities:
+http://www.elasticsearch.org/guide/reference/api/search/facets/
+
+There are various kinds of facets available, for example (full list on the facets page):
+
+* Terms_ - counts by distinct terms (values) in a field
+* Range_ - counts for a given set of ranges in a field
+* Histogram_ and `Date Histogram`_ - counts by constant interval ranges
+* Statistical_ - statistical summary of a field (mean, sum etc)
+* `Terms Stats`_ - statistical summary on one field (stats field) for distinct
+ terms in another field. For example, spending stats per department or per
+ region.
+* `Geo Distance`_: counts by distance ranges from a given point
+
+Note that you can apply multiple facets per query.
+
+.. _Terms: http://www.elasticsearch.org/guide/reference/api/search/facets/terms-facet.html
+.. _Range: http://www.elasticsearch.org/guide/reference/api/search/facets/range-facet.html
+.. _Histogram: http://www.elasticsearch.org/guide/reference/api/search/facets/histogram-facet.html
+.. _Date Histogram: http://www.elasticsearch.org/guide/reference/api/search/facets/date-histogram-facet.html
+.. _Statistical: http://www.elasticsearch.org/guide/reference/api/search/facets/statistical-facet.html
+.. _Terms Stats: http://www.elasticsearch.org/guide/reference/api/search/facets/terms-stats-facet.html
+.. _Geo Distance: http://www.elasticsearch.org/guide/reference/api/search/facets/geo-distance-facet.html
+
+
+Adding, Updating and Deleting Data
+==================================
+
+ElasticSeach, and hence the Data API, have a standard RESTful API. Thus::
+
+ POST {{endpoint}} : INSERT
+ PUT/POST {{endpoint}}/{{id}} : UPDATE (or INSERT)
+ DELETE {{endpoint}}/{{id}} : DELETE
+
+For more on INSERT and UPDATE see the `Index API`_ documentation.
+
+.. _Index API: http://elasticsearch.org/guide/reference/api/index_.html
+
+There is also support bulk insert and updates via the `Bulk API`_.
+
+.. _Bulk API: http://elasticsearch.org/guide/reference/api/bulk.html
Schema Mapping
diff --git a/setup.py b/setup.py
index 33f6d6b..cd37609 100644
--- a/setup.py
+++ b/setup.py
@@ -71,6 +71,7 @@
roles = ckan.lib.authztool:RolesCommand
celeryd = ckan.lib.cli:Celery
rdf-export = ckan.lib.cli:RDFExport
+ plugin-info = ckan.lib.cli:PluginInfo
[console_scripts]
ckan-admin = bin.ckan_admin:Command
================================================================
Commit: ff80476ae666b4dc9670274dcb101e1c0bfec074
https://github.com/okfn/ckan/commit/ff80476ae666b4dc9670274dcb101e1c0bfec074
Author: Ross Jones <rossdjones at gmail.com>
Date: 2012-04-24 (Tue, 24 Apr 2012)
Changed paths:
M ckan/lib/helpers.py
M ckan/templates/user/layout.html
M doc/_themes/sphinx-theme-okfn
Log Message:
-----------
Merge branch 'master' of https://github.com/okfn/ckan
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index dfdc0a8..78924b4 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -112,6 +112,9 @@ def _add_i18n_to_url(url_to_amend, **kw):
root = request.environ.get('SCRIPT_NAME', '')
except TypeError:
root = ''
+ if kw.get('qualified', False):
+ # if qualified is given we want the full url ie http://...
+ root = _routes_default_url_for('/', qualified=True)[:-1] + root
# ckan.root_path is defined when we have none standard language
# position in the url
root_path = config.get('ckan.root_path', None)
diff --git a/ckan/templates/user/layout.html b/ckan/templates/user/layout.html
index cf4d02c..2de8d35 100644
--- a/ckan/templates/user/layout.html
+++ b/ckan/templates/user/layout.html
@@ -14,7 +14,7 @@
</py:when>
<py:otherwise>
<py:if test="c.id">
- <li class="${'active' if c.action=='read' else ''}"><a href="${h.url_for(controller='user', action='me')}">View Profile</a></li>
+ <li class="${'active' if c.action=='read' else ''}"><a href="${h.url_for(controller='user', action='read', id=c.user_dict.name)}">View Profile</a></li>
</py:if>
<py:if test="not c.id">
<li class="${'active' if c.action=='login' else ''}"><a href="${h.url_for(controller='user', action='login')}">Login</a></li>
diff --git a/doc/_themes/sphinx-theme-okfn b/doc/_themes/sphinx-theme-okfn
index ef101a1..fd96c62 160000
--- a/doc/_themes/sphinx-theme-okfn
+++ b/doc/_themes/sphinx-theme-okfn
@@ -1 +1 @@
-Subproject commit ef101a18d6de959207361e233b89efd3be24e66f
+Subproject commit fd96c62b6426a3cc72b3c70314bc4af25e265cb3
================================================================
Commit: 79fb5fa7dc0d81529ebf57750fcb0ea601e205a1
https://github.com/okfn/ckan/commit/79fb5fa7dc0d81529ebf57750fcb0ea601e205a1
Author: Ross Jones <rossdjones at gmail.com>
Date: 2012-04-24 (Tue, 24 Apr 2012)
Changed paths:
M ckan/lib/base.py
M ckan/lib/dictization/model_save.py
M ckan/logic/schema.py
A ckan/migration/versions/054_add_resource_created_date.py
A ckan/migration/versions/055_update_user_and_activity_detail.py
M ckan/model/package.py
M ckan/model/resource.py
M ckan/public/scripts/application.js
M ckan/public/scripts/templates.js
M ckan/templates/js_strings.html
M ckan/tests/lib/test_dictization.py
Log Message:
-----------
Merge branch 'master' of https://github.com/okfn/ckan
diff --git a/ckan/lib/base.py b/ckan/lib/base.py
index dd1f03b..ffb79a7 100644
--- a/ckan/lib/base.py
+++ b/ckan/lib/base.py
@@ -53,16 +53,19 @@ def render_snippet(template_name, **kw):
comment tags added to show the template used. NOTE: unlike other
render functions this takes a list of keywords instead of a dict for
the extra template variables. '''
- output = render(template_name, extra_vars=kw)
+ # allow cache_force to be set in render function
+ cache_force = kw.pop('cache_force', None)
+ output = render(template_name, extra_vars=kw, cache_force=cache_force)
output = '\n<!-- Snippet %s start -->\n%s\n<!-- Snippet %s end -->\n' % (
template_name, output, template_name)
return literal(output)
-def render_text(template_name, extra_vars=None):
+def render_text(template_name, extra_vars=None, cache_force=None):
''' Helper function to render a genshi NewTextTemplate without
having to pass the loader_class or method. '''
return render(template_name,
extra_vars=extra_vars,
+ cache_force=cache_force,
method='text',
loader_class=NewTextTemplate)
diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py
index aa9f1a5..0d45b57 100644
--- a/ckan/lib/dictization/model_save.py
+++ b/ckan/lib/dictization/model_save.py
@@ -1,3 +1,4 @@
+import datetime
import uuid
from sqlalchemy.orm import class_mapper
import ckan.lib.dictization as d
@@ -8,7 +9,6 @@
def resource_dict_save(res_dict, context):
model = context["model"]
session = context["session"]
- trigger_url_change = False
id = res_dict.get("id")
obj = None
@@ -29,6 +29,9 @@ def resource_dict_save(res_dict, context):
if key in ('extras', 'revision_timestamp'):
continue
if key in fields:
+ if isinstance(getattr(obj, key), datetime.datetime):
+ if getattr(obj, key).isoformat() == value:
+ continue
if key == 'url' and not new and obj.url <> value:
obj.url_changed = True
setattr(obj, key, value)
diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py
index 0d41722..d756657 100644
--- a/ckan/logic/schema.py
+++ b/ckan/logic/schema.py
@@ -62,6 +62,7 @@ def default_resource_schema():
'webstore_url': [ignore_missing, unicode],
'cache_url': [ignore_missing, unicode],
'size': [ignore_missing, int_validator],
+ 'created': [ignore_missing, isodate],
'last_modified': [ignore_missing, isodate],
'cache_last_updated': [ignore_missing, isodate],
'webstore_last_updated': [ignore_missing, isodate],
diff --git a/ckan/migration/versions/054_add_resource_created_date.py b/ckan/migration/versions/054_add_resource_created_date.py
new file mode 100644
index 0000000..0364150
--- /dev/null
+++ b/ckan/migration/versions/054_add_resource_created_date.py
@@ -0,0 +1,9 @@
+def upgrade(migrate_engine):
+ migrate_engine.execute('''
+ ALTER TABLE resource
+ ADD COLUMN created timestamp without time zone;
+
+ ALTER TABLE resource_revision
+ ADD COLUMN created timestamp without time zone;
+ '''
+ )
diff --git a/ckan/migration/versions/055_update_user_and_activity_detail.py b/ckan/migration/versions/055_update_user_and_activity_detail.py
new file mode 100644
index 0000000..5d34900
--- /dev/null
+++ b/ckan/migration/versions/055_update_user_and_activity_detail.py
@@ -0,0 +1,9 @@
+def upgrade(migrate_engine):
+ migrate_engine.execute('''
+ ALTER TABLE activity_detail
+ ALTER COLUMN activity_id DROP NOT NULL;
+
+ ALTER TABLE "user"
+ ALTER COLUMN name SET NOT NULL;
+ '''
+ )
diff --git a/ckan/model/package.py b/ckan/model/package.py
index 4365963..dd1e0d3 100644
--- a/ckan/model/package.py
+++ b/ckan/model/package.py
@@ -114,6 +114,7 @@ def get_resource_identity(resource_obj_or_dict):
else:
resource = resource_obj_or_dict
res_dict = resource.as_dict(core_columns_only=True)
+ del res_dict['created']
return res_dict
existing_res_identites = [get_resource_identity(res) \
for res in self.resources]
diff --git a/ckan/model/resource.py b/ckan/model/resource.py
index 3d08575..f370e02 100644
--- a/ckan/model/resource.py
+++ b/ckan/model/resource.py
@@ -3,6 +3,7 @@
from sqlalchemy import orm
from pylons import config
import vdm.sqlalchemy
+import datetime
from meta import *
from types import make_uuid, JsonDictType
@@ -11,7 +12,7 @@
from ckan.model import extension
from ckan.model.activity import ActivityDetail
-__all__ = ['Resource', 'resource_table',
+__all__ = ['Resource', 'resource_table',
'ResourceGroup', 'resource_group_table',
'ResourceRevision', 'resource_revision_table',
'ResourceGroupRevision', 'resource_group_revision_table',
@@ -19,9 +20,9 @@
CORE_RESOURCE_COLUMNS = ['url', 'format', 'description', 'hash', 'name',
'resource_type', 'mimetype', 'mimetype_inner',
- 'size', 'last_modified', 'cache_url', 'cache_last_updated',
- 'webstore_url', 'webstore_last_updated']
-
+ 'size', 'created', 'last_modified', 'cache_url',
+ 'cache_last_updated', 'webstore_url',
+ 'webstore_last_updated']
##formally package_resource
@@ -40,6 +41,7 @@
Column('mimetype', types.UnicodeText),
Column('mimetype_inner', types.UnicodeText),
Column('size', types.BigInteger),
+ Column('created', types.DateTime, default=datetime.datetime.now),
Column('last_modified', types.DateTime),
Column('cache_url', types.UnicodeText),
Column('cache_last_updated', types.DateTime),
diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js
index d94bbd4..3c9b62d 100644
--- a/ckan/public/scripts/application.js
+++ b/ckan/public/scripts/application.js
@@ -638,6 +638,7 @@ CKAN.View.Resource = Backbone.View.extend({
word=='format' ||
word=='hash' ||
word=='id' ||
+ word=='created' ||
word=='last_modified' ||
word=='mimetype' ||
word=='mimetype_inner' ||
diff --git a/ckan/public/scripts/templates.js b/ckan/public/scripts/templates.js
index 66b06cb..6ecad11 100644
--- a/ckan/public/scripts/templates.js
+++ b/ckan/public/scripts/templates.js
@@ -143,6 +143,12 @@ CKAN.Templates.resourceDetails = ' \
</div> \
</div> \
<div class="control-group"> \
+ <label for="" class="control-label" property="rdfs:label">'+CKAN.Strings.created+'</label> \
+ <div class="controls"> \
+ <input type="text" disabled="disabled" value="${resource.created}" class="disabled" /> \
+ </div> \
+ </div> \
+ <div class="control-group"> \
<label class="control-label">'+CKAN.Strings.extraFields+' \
<button class="btn btn-small add-resource-extra">'+CKAN.Strings.addExtraField+'</button>\
</label>\
diff --git a/ckan/templates/js_strings.html b/ckan/templates/js_strings.html
index 5ba337c..76a0d7e 100644
--- a/ckan/templates/js_strings.html
+++ b/ckan/templates/js_strings.html
@@ -51,6 +51,7 @@
CKAN.Strings.datastoreEnabled = "${_('DataStore enabled')}";
CKAN.Strings.sizeBracketsBytes = "${_('Size (Bytes)')}";
CKAN.Strings.mimetype = "${_('Mimetype')}";
+ CKAN.Strings.created = "${_('Created')}";
CKAN.Strings.lastModified = "${_('Last Modified')}";
CKAN.Strings.mimetypeInner = "${_('Mimetype (Inner)')}";
CKAN.Strings.hash = "${_('Hash')}";
diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py
index 09342d2..d32fd89 100644
--- a/ckan/tests/lib/test_dictization.py
+++ b/ckan/tests/lib/test_dictization.py
@@ -353,7 +353,7 @@ def test_08_package_save(self):
def test_09_package_alter(self):
context = {"model": model,
- "session": model.Session}
+ "session": model.Session}
anna1 = model.Session.query(model.Package).filter_by(name='annakarenina').one()
================================================================
Commit: 6799e695f97c0019742fc2fa6dc6239af357faf1
https://github.com/okfn/ckan/commit/6799e695f97c0019742fc2fa6dc6239af357faf1
Author: Ross Jones <rossdjones at gmail.com>
Date: 2012-04-24 (Tue, 24 Apr 2012)
Changed paths:
M ckan/controllers/user.py
Log Message:
-----------
[xs] Fix for GET path through password reset
diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py
index 7f46b4f..d17df6f 100644
--- a/ckan/controllers/user.py
+++ b/ckan/controllers/user.py
@@ -383,6 +383,8 @@ def perform_reset(self, id):
h.flash_error(u'%r'% e.error_dict)
except ValueError, ve:
h.flash_error(unicode(ve))
+
+ c.user_dict = user_dict
return render('user/perform_reset.html')
def _format_about(self, about):
================================================================
Compare: https://github.com/okfn/ckan/compare/ffec1e6...6799e69
More information about the ckan-changes
mailing list