[ckan-changes] commit/ckan: 3 new changesets
Bitbucket
commits-noreply at bitbucket.org
Tue Nov 8 17:50:23 UTC 2011
3 new commits in ckan:
https://bitbucket.org/okfn/ckan/changeset/511770d592a0/
changeset: 511770d592a0
user: dread
date: 2011-11-04 18:28:00
summary: [bin]: OKF Task #933 - script for copying datasets between ckan instances, using api.
affected #: 1 file
diff -r 97e1e90d66d745f44f4ce4a902640efef659d695 -r 511770d592a03ffffe545461ace1537b74c8802c bin/copy-ckan-2-ckan.py
--- /dev/null
+++ b/bin/copy-ckan-2-ckan.py
@@ -0,0 +1,121 @@
+import ckanclient
+from optparse import OptionParser
+import re
+
+# Script for copying packages from one CKAN instance to another using the API
+# for both.
+
+# Some of the commands used:
+# $ pg_dump -f dump_ckan/nederland.ckan.net.pg_dump -U okfn nederland
+# $ rsync --progress -z okfn at eu5.okfn.org:/home/okfn/dump_ckan/* /home/dread/db_backup/communities/
+# $ paster db clean && paster db load /home/dread/db_backup/communities/nederland.ckan.net.pg_dump
+# $ python bin/copy-ckan-2-ckan.py -k tester -g country-si -t "Slovenia" -s si.ckan.net http://127.0.0.1:5000/api http://127.0.0.1:5001/api
+
+def copy_packages(source_ckan_uri,
+ dest_ckan_uri, dest_api_key,
+ dest_group_name, dest_group_title,
+ site_name):
+ ckan1 = ckanclient.CkanClient(base_location=source_ckan_uri)
+ ckan2 = ckanclient.CkanClient(base_location=dest_ckan_uri,
+ api_key=dest_api_key)
+
+ # ensure pkg references will be the same type
+ ckan1_api_version = ckan1.api_version_get()
+ ckan2_api_version = ckan2.api_version_get()
+ assert ckan1_api_version == ckan2_api_version
+
+ # ensure group exists
+ existing_groups = set(ckan2.group_register_get())
+ def add_group_if_needed(group_name, group_title=None, fetch_title=False):
+ if group_name not in existing_groups:
+ if fetch_title:
+ group_title = ckan1.group_entity_get(group_name)['title']
+ group = {'name': group_name,
+ 'title': group_title}
+ ckan2.group_register_post(group)
+ existing_groups.add(group_name)
+ print 'Created group: %s' % group_name
+
+ if dest_group_name:
+ add_group_if_needed(dest_group_name, dest_group_title)
+
+ existing_pkgs = ckan2.package_register_get()
+
+ if site_name:
+ import_tag = 'meta.imported-from.%s' % re.sub('[^a-zA-Z0-9-_.]', '', site_name)
+ print 'Tagging with: %s' % import_tag
+ else:
+ import_tag = None
+
+ # go through all packages
+ package_list = ckan1.package_register_get()
+ print 'Found %i packages to copy' % len(package_list)
+ for package_ref in package_list[:]:
+ try:
+ pkg = ckan1.package_entity_get(package_ref)
+ except ckanclient.CkanApiNotAuthorizedError:
+ print '!! Not authorized: %s' % package_ref
+ package_list.remove(package_ref)
+ continue
+ print 'Got package: %s' % pkg['name']
+
+ # put in groups
+ for group_name in pkg['groups']:
+ add_group_if_needed(group_name, fetch_title=True)
+ if dest_group_name:
+ # only works on CKAN 1.5.1 so add all at end anyway
+ pkg['groups'].append(dest_group_name)
+
+ del pkg['id']
+
+ if import_tag:
+ pkg['tags'].append(import_tag)
+
+ # do the copy
+ if package_ref in existing_pkgs:
+ existing_pkg = ckan2.package_entity_get(pkg['name'])
+ pkg_returned = ckan2.package_entity_put(pkg)
+ print '...updated'
+ else:
+ pkg_returned = ckan2.package_register_post(pkg)
+ print '...created'
+ groups_not_added_to = set(pkg['groups']) - set(pkg_returned['groups']) - set((dest_group_name))
+ # packages don't get added to the group before when CKAN <= 1.5 so
+ # we have to do this now
+ for group_ref in groups_not_added_to:
+ group = ckan2.group_entity_get(group_ref)
+ group['packages'].append(pkg['name'])
+ ckan2.group_entity_put(group)
+ print '...and added to group %s' % group_ref
+
+ group = ckan2.group_entity_get(dest_group_name)
+ pkgs_to_add_to_group = list(set(package_list) - set(group['packages']))
+ if pkgs_to_add_to_group:
+ print 'Adding %i packages to group %s: %r' % (len(pkgs_to_add_to_group), dest_group_name, pkgs_to_add_to_group)
+ group['packages'].extend(pkgs_to_add_to_group)
+ ckan2.group_entity_put(group)
+
+usage = '''%prog [OPTIONS] <source_ckan_api_uri><destination_ckan_api_uri>
+Copy datasets from ckan to another and put them in a group.'''
+parser = OptionParser(usage=usage)
+parser.add_option("-k", "--destination-ckan-api-key", dest="destination_ckan_api_key",
+ help="Destination CKAN's API key", metavar="API-KEY")
+parser.add_option("-g", "--group-name", dest="group_name",
+ help="Destination CKAN group's name")
+parser.add_option("-t", "--group-title", dest="group_title",
+ help="Destination CKAN group's title")
+parser.add_option("-s", "--site-name", dest="site_name",
+ help="Name of source CKAN site - so source can be tagged")
+
+(options, args) = parser.parse_args()
+
+assert len(args) == 2, 'The source and destination CKAN API URIs are the only two arguments. Found: %r' % args
+source_ckan_uri, destination_ckan_uri = args
+print 'Key: ', options.destination_ckan_api_key
+
+copy_packages(source_ckan_uri,
+ destination_ckan_uri,
+ options.destination_ckan_api_key,
+ options.group_name, options.group_title,
+ options.site_name)
+
https://bitbucket.org/okfn/ckan/changeset/91ebbb22956a/
changeset: 91ebbb22956a
user: dread
date: 2011-11-04 18:30:18
summary: [controllers, tests][xs]: More info on search index errors to track down v. occasional issue. Renamed test.
affected #: 2 files
diff -r 511770d592a03ffffe545461ace1537b74c8802c -r 91ebbb22956ac649242b91f022a2c143310f820d ckan/controllers/package.py
--- a/ckan/controllers/package.py
+++ b/ckan/controllers/package.py
@@ -452,8 +452,8 @@
abort(404, _('Package not found'))
except DataError:
abort(400, _(u'Integrity Error'))
- except SearchIndexError:
- abort(500, _(u'Unable to add package to search index.'))
+ except SearchIndexError, e:
+ abort(500, _(u'Unable to add package to search index.') + repr(e.args))
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
@@ -481,8 +481,8 @@
abort(404, _('Package not found'))
except DataError:
abort(400, _(u'Integrity Error'))
- except SearchIndexError:
- abort(500, _(u'Unable to update search index.'))
+ except SearchIndexError, e:
+ abort(500, _(u'Unable to update search index.') + repr(e.args))
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
diff -r 511770d592a03ffffe545461ace1537b74c8802c -r 91ebbb22956ac649242b91f022a2c143310f820d ckan/tests/functional/api/test_package_search.py
--- a/ckan/tests/functional/api/test_package_search.py
+++ b/ckan/tests/functional/api/test_package_search.py
@@ -90,7 +90,7 @@
self.assert_results(res_dict, ['testpkg'])
assert res_dict['count'] == 1, res_dict['count']
- def test_04_post_qjson(self):
+ def test_04_post_json(self):
query = {'q': self.package_fixture_data['name']}
json_query = self.dumps(query)
offset = self.base_url
@@ -99,7 +99,7 @@
self.assert_results(res_dict, ['testpkg'])
assert res_dict['count'] == 1, res_dict['count']
- def test_05_uri_qjson_tags(self):
+ def test_05_uri_json_tags(self):
query = {'q': 'annakarenina tags:russian tags:tolstoy'}
json_query = self.dumps(query)
offset = self.base_url + '?qjson=%s' % json_query
@@ -108,7 +108,7 @@
self.assert_results(res_dict, [u'annakarenina'])
assert res_dict['count'] == 1, res_dict
- def test_05_uri_qjson_tags_multiple(self):
+ def test_05_uri_json_tags_multiple(self):
query = {'q': 'tags:russian tags:tolstoy'}
json_query = self.dumps(query)
offset = self.base_url + '?qjson=%s' % json_query
https://bitbucket.org/okfn/ckan/changeset/01e3f111cf6a/
changeset: 01e3f111cf6a
user: dread
date: 2011-11-08 18:50:07
summary: [cli][xs]: Improved poor error message for when config file cannot be found.
affected #: 2 files
diff -r 91ebbb22956ac649242b91f022a2c143310f820d -r 01e3f111cf6aceaa29d42306ec3903533a166cb7 ckan/lib/cli.py
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -38,6 +38,8 @@
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
+ if not os.path.exists(self.filename):
+ raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
diff -r 91ebbb22956ac649242b91f022a2c143310f820d -r 01e3f111cf6aceaa29d42306ec3903533a166cb7 doc/common-error-messages.rst
--- a/doc/common-error-messages.rst
+++ b/doc/common-error-messages.rst
@@ -107,4 +107,9 @@
pip install -e hg+https://hg.3aims.com/public/ApacheMiddleware@tip#egg=apachemiddleware
-And it should complete this time.
\ No newline at end of file
+And it should complete this time.
+
+``ConfigParser.NoSectionError: No section: 'formatters'``
+=========================================================
+
+This suggests that the config file specified with the paster ``--config`` parameter (e.g. ``myconfig.ini``) is incorrectly formatted. This may be true, but this error is also printed if you specify an incorrect filename for the config file!
\ No newline at end of file
Repository URL: https://bitbucket.org/okfn/ckan/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list