[ckan-changes] commit/ckanext-harvest: 4 new changesets
Bitbucket
commits-noreply at bitbucket.org
Fri Nov 18 14:35:59 UTC 2011
4 new commits in ckanext-harvest:
https://bitbucket.org/okfn/ckanext-harvest/changeset/3a3dc0e4c7eb/
changeset: 3a3dc0e4c7eb
branch: feature-new-ckan-harvester-features
user: amercader
date: 2011-11-18 14:20:41
summary: [ckan harvester] Support for default tags and groups
affected #: 3 files
diff -r 3b95a87964c0721f1b3f8bab1feaaaa658e7b58f -r 3a3dc0e4c7eba2ee815a19f79e5c2049f30812d4 README.rst
--- a/README.rst
+++ b/README.rst
@@ -113,6 +113,30 @@
After adding it, a 'CKAN' option should appear in the 'New harvest source' form.
+The CKAN harvesters support a number of configuration options to control their
+behaviour. Those need to defined as a JSON object in the configuration form
+field. The currently supported configuration options are:
+
+* api_version: You can force the harvester to use eithoer version '1' or
+ '2' of the CKAN API. Default is '2'.
+
+* default_tags: A list of tags that will be added to all harvested datasets.
+ Tags don't need to previously exist.
+
+* default_groups: A list of groups to which the harvested datasets will be
+ added to. The groups must exist. Note that you must use ids or names to
+ define the groups according to the API version you defined (names for
+ version '1', ids for version '2')
+
+Here is an example of a configuration object (the one that must be entered in
+the configuration field)::
+
+ {
+ "api_version":"1",
+ "default_tags":["new-tag-1","new-tag-2"],
+ "default_groups":["my-own-group"]
+ }
+
The harvesting interface
========================
diff -r 3b95a87964c0721f1b3f8bab1feaaaa658e7b58f -r 3a3dc0e4c7eba2ee815a19f79e5c2049f30812d4 ckanext/harvest/harvesters/base.py
--- a/ckanext/harvest/harvesters/base.py
+++ b/ckanext/harvest/harvesters/base.py
@@ -106,11 +106,17 @@
schema = default_package_schema()
schema["id"] = [ignore_missing, unicode]
+ # Check API version
+ if self.config:
+ api_version = self.config.get('api_version','2')
+ else:
+ api_verion = '2'
+
context = {
'model': model,
'session': Session,
'user': u'harvest',
- 'api_version':'2',
+ 'api_version': api_version,
'schema': schema,
}
diff -r 3b95a87964c0721f1b3f8bab1feaaaa658e7b58f -r 3a3dc0e4c7eba2ee815a19f79e5c2049f30812d4 ckanext/harvest/harvesters/ckanharvester.py
--- a/ckanext/harvest/harvesters/ckanharvester.py
+++ b/ckanext/harvest/harvesters/ckanharvester.py
@@ -1,7 +1,9 @@
import urllib2
+from ckan.lib.base import c
+from ckan import model
from ckan.model import Session, Package
-from ckan.logic import ValidationError, NotFound
+from ckan.logic import ValidationError, NotFound, get_action
from ckan.lib.helpers import json
from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError, \
@@ -65,6 +67,16 @@
try:
config_obj = json.loads(config)
+
+ if 'default_groups' in config_obj:
+ # Check if default groups exist
+ context = {'model':model,'user':c.user}
+ for group_name in config_obj['default_groups']:
+ try:
+ group = get_action('group_show')(context,{'id':group_name})
+ except NotFound,e:
+ raise ValueError('Default group not found')
+
except ValueError,e:
raise e
@@ -196,6 +208,24 @@
try:
package_dict = json.loads(harvest_object.content)
+
+ # Set default tags if needed
+ default_tags = self.config.get('default_tags',[])
+ if default_tags:
+ if not 'tags' in package_dict:
+ package_dict['tags'] = []
+ package_dict['tags'].extend([t for t in default_tags if t not in package_dict['tags']])
+
+ # Ignore remote groups for the time being
+ del package_dict['groups']
+
+ # Set default groups if needed
+ default_groups = self.config.get('default_groups',[])
+ if default_groups:
+ if not 'groups' in package_dict:
+ package_dict['groups'] = []
+ package_dict['groups'].extend([g for g in default_groups if g not in package_dict['groups']])
+
return self._create_or_update_package(package_dict,harvest_object)
except ValidationError,e:
self._save_object_error('Invalid package with GUID %s: %r' % (harvest_object.guid, e.error_dict),
https://bitbucket.org/okfn/ckanext-harvest/changeset/2863992f5a93/
changeset: 2863992f5a93
branch: feature-new-ckan-harvester-features
user: amercader
date: 2011-11-18 15:12:30
summary: [ckan harvester] Support for defining a custom user to do the harvesting
affected #: 3 files
diff -r 3a3dc0e4c7eba2ee815a19f79e5c2049f30812d4 -r 2863992f5a93773190b131aef7943c2df9afd8d1 README.rst
--- a/README.rst
+++ b/README.rst
@@ -117,7 +117,7 @@
behaviour. Those need to defined as a JSON object in the configuration form
field. The currently supported configuration options are:
-* api_version: You can force the harvester to use eithoer version '1' or
+* api_version: You can force the harvester to use either version '1' or
'2' of the CKAN API. Default is '2'.
* default_tags: A list of tags that will be added to all harvested datasets.
@@ -128,13 +128,18 @@
define the groups according to the API version you defined (names for
version '1', ids for version '2')
+* user: User who will run the harvesting process. Please note that this user
+ needs to have permission for creating packages, and if default groups were
+ defined, the user must have permission to assign packages to these groups.
+
Here is an example of a configuration object (the one that must be entered in
the configuration field)::
{
"api_version":"1",
"default_tags":["new-tag-1","new-tag-2"],
- "default_groups":["my-own-group"]
+ "default_groups":["my-own-group"],
+ "user":"harverster-user"
}
diff -r 3a3dc0e4c7eba2ee815a19f79e5c2049f30812d4 -r 2863992f5a93773190b131aef7943c2df9afd8d1 ckanext/harvest/harvesters/base.py
--- a/ckanext/harvest/harvesters/base.py
+++ b/ckanext/harvest/harvesters/base.py
@@ -109,13 +109,16 @@
# Check API version
if self.config:
api_version = self.config.get('api_version','2')
+ #TODO: use site user when available
+ user_name = self.config.get('user',u'harvest')
else:
api_verion = '2'
+ user_name = u'harvest'
context = {
'model': model,
'session': Session,
- 'user': u'harvest',
+ 'user': user_name,
'api_version': api_version,
'schema': schema,
}
diff -r 3a3dc0e4c7eba2ee815a19f79e5c2049f30812d4 -r 2863992f5a93773190b131aef7943c2df9afd8d1 ckanext/harvest/harvesters/ckanharvester.py
--- a/ckanext/harvest/harvesters/ckanharvester.py
+++ b/ckanext/harvest/harvesters/ckanharvester.py
@@ -77,6 +77,14 @@
except NotFound,e:
raise ValueError('Default group not found')
+ if 'user' in config_obj:
+ # Check if user exists
+ context = {'model':model,'user':c.user}
+ try:
+ user = get_action('user_show')(context,{'id':config_obj.get('user')})
+ except NotFound,e:
+ raise ValueError('User not found')
+
except ValueError,e:
raise e
https://bitbucket.org/okfn/ckanext-harvest/changeset/2c9f98206e73/
changeset: 2c9f98206e73
branch: feature-new-ckan-harvester-features
user: amercader
date: 2011-11-18 15:30:10
summary: [ckan harvester] Support for creating read-only packages
affected #: 2 files
diff -r 2863992f5a93773190b131aef7943c2df9afd8d1 -r 2c9f98206e735c8da203ce673b5c6c9eaaf0684f README.rst
--- a/README.rst
+++ b/README.rst
@@ -132,6 +132,12 @@
needs to have permission for creating packages, and if default groups were
defined, the user must have permission to assign packages to these groups.
+* read_only: Create harvested packages in read-only mode. Only the user who
+ performed the harvest (the one defined in the previous setting or the
+ 'harvest' sysadmin) will be able to edit and administer the packages
+ created from this harvesting source. Logged in users and visitors will be
+ only able to read them.
+
Here is an example of a configuration object (the one that must be entered in
the configuration field)::
@@ -139,7 +145,8 @@
"api_version":"1",
"default_tags":["new-tag-1","new-tag-2"],
"default_groups":["my-own-group"],
- "user":"harverster-user"
+ "user":"harverster-user",
+ "read_only": true
}
@@ -288,7 +295,7 @@
pending harvesting jobs::
paster harvester run --config=../ckan/development.ini
-
+
After packages have been imported, the search index will have to be updated
before the packages appear in search results (from the ckan directory):
diff -r 2863992f5a93773190b131aef7943c2df9afd8d1 -r 2c9f98206e735c8da203ce673b5c6c9eaaf0684f ckanext/harvest/harvesters/ckanharvester.py
--- a/ckanext/harvest/harvesters/ckanharvester.py
+++ b/ckanext/harvest/harvesters/ckanharvester.py
@@ -234,7 +234,26 @@
package_dict['groups'] = []
package_dict['groups'].extend([g for g in default_groups if g not in package_dict['groups']])
- return self._create_or_update_package(package_dict,harvest_object)
+ result = self._create_or_update_package(package_dict,harvest_object)
+
+ if result and self.config.get('read_only',False) == True:
+
+ package = model.Package.get(package_dict['id'])
+
+ # Clear default permissions
+ model.clear_user_roles(package)
+
+ # Setup harvest user as admin
+ user_name = self.config.get('user',u'harvest')
+ user = model.User.get(user_name)
+ pkg_role = model.PackageRole(package=package, user=user, role=model.Role.ADMIN)
+
+ # Other users can only read
+ for user_name in (u'visitor',u'logged_in'):
+ user = model.User.get(user_name)
+ pkg_role = model.PackageRole(package=package, user=user, role=model.Role.READER)
+
+
except ValidationError,e:
self._save_object_error('Invalid package with GUID %s: %r' % (harvest_object.guid, e.error_dict),
harvest_object, 'Import')
https://bitbucket.org/okfn/ckanext-harvest/changeset/de092a858798/
changeset: de092a858798
branch: feature-new-ckan-harvester-features
user: amercader
date: 2011-11-18 15:35:46
summary: [ui] Show config options in harvest source details page
affected #: 1 file
diff -r 2c9f98206e735c8da203ce673b5c6c9eaaf0684f -r de092a858798cd63af9e4d5d8c1bd6925b7722bc ckanext/harvest/templates/source/read.html
--- a/ckanext/harvest/templates/source/read.html
+++ b/ckanext/harvest/templates/source/read.html
@@ -35,6 +35,15 @@
<td>${c.source.description}</td></tr><tr>
+ <th>Configuration</th>
+ <py:if test="c.source.config">
+ <td>${c.source.config}</td>
+ </py:if>
+ <py:if test="not c.source.config">
+ <td>-</td>
+ </py:if>
+ </tr>
+ <tr><th>User</th><td>${c.source.user_id}</td></tr>
Repository URL: https://bitbucket.org/okfn/ckanext-harvest/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list