[ckan-changes] commit/ckan: 3 new changesets
Bitbucket
commits-noreply at bitbucket.org
Tue May 10 11:01:59 UTC 2011
3 new changesets in ckan:
http://bitbucket.org/okfn/ckan/changeset/70cb8df12d14/
changeset: r3066:70cb8df12d14
branch: feature-1078-refactor-wui-to-use-logic-layer
user: kindly
date: 2011-05-06 19:03:11
summary: [merge] default
affected #: 23 files (54.5 KB)
--- a/.hgignore Fri May 06 18:00:57 2011 +0100
+++ b/.hgignore Fri May 06 18:03:11 2011 +0100
@@ -27,3 +27,6 @@
.noseids
*~
+# nosetest coverage output
+.coverage
+htmlcov/*
--- a/CHANGELOG.txt Fri May 06 18:00:57 2011 +0100
+++ b/CHANGELOG.txt Fri May 06 18:03:11 2011 +0100
@@ -1,6 +1,12 @@
CKAN CHANGELOG
++++++++++++++
+v1.3.5 2011-XX-XX
+=================
+Major:
+ * Authorization forms now in grid format (#1074)
+ * Links to RDF, N3 and Turtle metadata formats provided by semantic.ckan.net (#1088)
+
v1.3.4 2011-XX-XX
=================
Major:
--- a/ckan/config/routing.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/config/routing.py Fri May 06 18:03:11 2011 +0100
@@ -194,9 +194,10 @@
map.connect('/group/list', controller='group', action='list')
map.connect('/group/new', controller='group', action='new')
map.connect('/group/{action}/{id}', controller='group',
- requirements=dict(actions='|'.join([
+ requirements=dict(action='|'.join([
'edit',
- 'authz'
+ 'authz',
+ 'history'
]))
)
map.connect('/group/{id}', controller='group', action='read')
@@ -227,13 +228,13 @@
map.connect('/user/logged_out', controller='user', action='logged_out')
map.connect('/user/me', controller='user', action='me')
map.connect('/user/{id:.*}', controller='user', action='read')
- map.connect('/user', controller='revision', action='index')
+ map.connect('/user', controller='user', action='index')
-
-
- map.connect('/revision/{action}/{id}', controller='revision')
- map.connect('/revision/{action}', controller='revision')
map.connect('/revision', controller='revision', action='index')
+ map.connect('/revision/edit/{id}', controller='revision', action='edit')
+ map.connect('/revision/diff/{id}', controller='revision', action='diff')
+ map.connect('/revision/list', controller='revision', action='list')
+ map.connect('/revision/{id}', controller='revision', action='read')
for plugin in routing_plugins:
map = plugin.after_map(map)
--- a/ckan/controllers/api.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/controllers/api.py Fri May 06 18:03:11 2011 +0100
@@ -1,6 +1,8 @@
import logging
from paste.util.multidict import MultiDict
+from webob.multidict import UnicodeMultiDict
+
from ckan.lib.base import BaseController, response, c, _, gettext, request
from ckan.lib.helpers import json
import ckan.model as model
@@ -327,20 +329,11 @@
revs = model.Session.query(model.Revision).filter(model.Revision.timestamp>since_time)
return self._finish_ok([rev.id for rev in revs])
elif register == 'package' or register == 'resource':
- if request.params.has_key('qjson'):
- if not request.params['qjson']:
- response.status_int = 400
- return gettext('Blank qjson parameter')
- params = json.loads(request.params['qjson'])
- elif request.params.values() and request.params.values() != [u''] and request.params.values() != [u'1']:
- params = request.params
- else:
- try:
- params = self._get_request_data()
- except ValueError, inst:
- response.status_int = 400
- return gettext(u'Search params: %s') % unicode(inst)
-
+ try:
+ params = self._get_search_params(request.params)
+ except ValueError, e:
+ response.status_int = 400
+ return gettext('Could not read parameters: %r' % e)
options = QueryOptions()
for k, v in params.items():
if (k in DEFAULT_OPTIONS.keys()):
@@ -382,6 +375,24 @@
response.status_int = 404
return gettext('Unknown register: %s') % register
+ @classmethod
+ def _get_search_params(cls, request_params):
+ if request_params.has_key('qjson'):
+ try:
+ params = json.loads(request_params['qjson'], encoding='utf8')
+ except ValueError, e:
+ raise ValueError, gettext('Malformed qjson value') + ': %r' % e
+ elif len(request_params) == 1 and \
+ len(request_params.values()[0]) < 2 and \
+ request_params.keys()[0].startswith('{'):
+ # e.g. {some-json}='1' or {some-json}=''
+ params = json.loads(request_params.keys()[0], encoding='utf8')
+ else:
+ params = request_params
+ if not isinstance(params, (UnicodeMultiDict, dict)):
+ raise ValueError, _('Request params must be in form of a json encoded dictionary.')
+ return params
+
def tag_counts(self, ver=None):
log.debug('tag counts')
tags = model.Session.query(model.Tag).all()
--- a/ckan/controllers/authorization_group.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/controllers/authorization_group.py Fri May 06 18:03:11 2011 +0100
@@ -77,7 +77,7 @@
model.setup_default_user_roles(authorization_group, [user])
users = [model.User.by_name(name) for name in \
request.params.getall('AuthorizationGroup-users-current')]
- authorization_group.users = list(set(users + [user]))
+ authorization_group.users = list(set(users))
usernames = request.params.getall('AuthorizationGroupUser--user_name')
for username in usernames:
if username:
@@ -131,7 +131,7 @@
user = model.User.by_name(c.user)
users = [model.User.by_name(name) for name in \
request.params.getall('AuthorizationGroup-users-current')]
- authorization_group.users = list(set(users + [user]))
+ authorization_group.users = list(set(users))
usernames = request.params.getall('AuthorizationGroupUser--user_name')
for username in usernames:
if username:
--- a/ckan/controllers/error.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/controllers/error.py Fri May 06 18:03:11 2011 +0100
@@ -26,7 +26,7 @@
original_request = request.environ.get('pylons.original_request')
original_response = request.environ.get('pylons.original_response')
# Bypass error template for API operations.
- if original_request.path.startswith('/api'):
+ if original_request and original_request.path.startswith('/api'):
return original_response.body
# Otherwise, decorate original response with error template.
c.content = literal(original_response.unicode_body) or cgi.escape(request.GET.get('message', ''))
--- a/ckan/controllers/package.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/controllers/package.py Fri May 06 18:03:11 2011 +0100
@@ -46,8 +46,6 @@
q = c.q = request.params.get('q') # unicode format (decoded from utf8)
c.open_only = request.params.get('open_only')
c.downloadable_only = request.params.get('downloadable_only')
- if c.q is None or len(c.q.strip()) == 0:
- q = '*:*'
c.query_error = False
try:
page = int(request.params.get('page', 1))
--- a/ckan/lib/base.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/lib/base.py Fri May 06 18:03:11 2011 +0100
@@ -134,28 +134,39 @@
def _get_tag(self, reference):
return model.Tag.get(reference)
- def _get_request_data(self):
- self.log.debug('Retrieving request params: %r' % request.params)
- self.log.debug('Retrieving request POST: %r' % request.POST)
+ @classmethod
+ def _get_request_data(cls):
+ '''Returns a dictionary, extracted from a request. The request data is
+ in POST data and formatted as a dictionary that has been
+ JSON-encoded.
+
+ If there is no data, None or "" is returned.
+ ValueError will be raised if the data is not a JSON-formatted dict.
+
+ '''
+ cls.log.debug('Retrieving request params: %r' % request.params)
+ cls.log.debug('Retrieving request POST: %r' % request.POST)
try:
- request_data = request.POST.keys()[0]
+ request_data = request.POST.keys()
except Exception, inst:
- msg = _("Can't find entity data in request POST data %s: %s") % (
- request.POST, str(inst)
- )
+ msg = _("Could not find the POST data: %r : %s") % \
+ (request.POST, inst)
raise ValueError, msg
- request_data = json.loads(request_data, encoding='utf8')
- if not isinstance(request_data, dict):
- raise ValueError, _("Request params must be in form of a json encoded dictionary.")
- # ensure unicode values
- for key, val in request_data.items():
- # if val is str then assume it is ascii, since json converts
- # utf8 encoded JSON to unicode
- request_data[key] = self._make_unicode(val)
- self.log.debug('Request data extracted: %r' % request_data)
+ if request_data:
+ request_data = request_data[0]
+ request_data = json.loads(request_data, encoding='utf8')
+ if not isinstance(request_data, dict):
+ raise ValueError, _("Request params must be in form of a json encoded dictionary.")
+ # ensure unicode values
+ for key, val in request_data.items():
+ # if val is str then assume it is ascii, since json converts
+ # utf8 encoded JSON to unicode
+ request_data[key] = cls._make_unicode(val)
+ cls.log.debug('Request data extracted: %r' % request_data)
return request_data
-
- def _make_unicode(self, entity):
+
+ @classmethod
+ def _make_unicode(cls, entity):
"""Cast bare strings and strings in lists or dicts to Unicode
"""
if isinstance(entity, str):
@@ -163,12 +174,12 @@
elif isinstance(entity, list):
new_items = []
for item in entity:
- new_items.append(self._make_unicode(item))
+ new_items.append(cls._make_unicode(item))
return new_items
elif isinstance(entity, dict):
new_dict = {}
for key, val in entity.items():
- new_dict[key] = self._make_unicode(val)
+ new_dict[key] = cls._make_unicode(val)
return new_dict
else:
return entity
--- a/ckan/lib/cli.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/lib/cli.py Fri May 06 18:03:11 2011 +0100
@@ -212,17 +212,19 @@
'''Creates a search index for all packages
Usage:
- search-index rebuild - indexes all packages
+ search-index rebuild - indexes all packages (default)
+ search-index check - checks for packages not indexed
+ search-index show {package-name} - shows index of a package
'''
summary = __doc__.split('\n')[0]
usage = __doc__
- max_args = 1
+ max_args = 2
min_args = 0
def command(self):
self._load_config()
- from ckan.lib.search import rebuild, check
+ from ckan.lib.search import rebuild, check, show
if not self.args:
# default to run
@@ -234,6 +236,11 @@
rebuild()
elif cmd == 'check':
check()
+ elif cmd == 'show':
+ if not len(self.args) == 2:
+ import pdb; pdb.set_trace()
+ self.args
+ show(self.args[1])
else:
print 'Command %s not recognized' % cmd
--- a/ckan/lib/create_test_data.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/lib/create_test_data.py Fri May 06 18:03:11 2011 +0100
@@ -377,10 +377,10 @@
# authz
model.Session.add_all([
model.User(name=u'tester', apikey=u'tester', password=u'tester'),
- model.User(name=u'joeadmin'),
- model.User(name=u'annafan', about=u'I love reading Annakarenina'),
- model.User(name=u'russianfan'),
- model.User(name=u'testsysadmin'),
+ model.User(name=u'joeadmin', password=u'joeadmin'),
+ model.User(name=u'annafan', about=u'I love reading Annakarenina', password=u'annafan'),
+ model.User(name=u'russianfan', password=u'russianfan'),
+ model.User(name=u'testsysadmin', password=u'testsysadmin'),
])
cls.user_refs.extend([u'tester', u'joeadmin', u'annafan', u'russianfan', u'testsysadmin'])
model.repo.commit_and_remove()
@@ -407,6 +407,33 @@
from ckan.model.changeset import ChangesetRegister
changeset_ids = ChangesetRegister().commit()
+ # Create a couple of authorization groups
+ for ag_name in [u'anauthzgroup', u'anotherauthzgroup']:
+ ag=model.AuthorizationGroup.by_name(ag_name)
+ if not ag: #may already exist, if not create
+ ag=model.AuthorizationGroup(name=ag_name)
+ model.Session.add(ag)
+
+ model.repo.commit_and_remove()
+
+ # and give them a range of roles on various things
+ ag = model.AuthorizationGroup.by_name(u'anauthzgroup')
+ aag = model.AuthorizationGroup.by_name(u'anotherauthzgroup')
+ pkg = model.Package.by_name(u'warandpeace')
+ g = model.Group.by_name('david')
+
+ model.add_authorization_group_to_role(ag, u'editor', model.System())
+ model.add_authorization_group_to_role(ag, u'reader', pkg)
+ model.add_authorization_group_to_role(ag, u'admin', aag)
+ model.add_authorization_group_to_role(aag, u'editor', ag)
+ model.add_authorization_group_to_role(ag, u'editor', g)
+
+ model.repo.commit_and_remove()
+
+
+
+
+
@classmethod
def create_user(cls, name='', **kwargs):
import ckan.model as model
--- a/ckan/lib/search/__init__.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/lib/search/__init__.py Fri May 06 18:03:11 2011 +0100
@@ -63,6 +63,12 @@
pkg = model.Session.query(model.Package).get(pkg_id)
print pkg.revision.timestamp.strftime('%Y-%m-%d'), pkg.name
+def show(package_reference):
+ from ckan import model
+ backend = get_backend()
+ package_index = backend.index_for(model.Package)
+ print package_index.get_index(package_reference)
+
def query_for(_type, backend=None):
""" Query for entities of a specified type (name, class, instance). """
return get_backend(backend=backend).query_for(_type)
--- a/ckan/lib/search/common.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/lib/search/common.py Fri May 06 18:03:11 2011 +0100
@@ -1,6 +1,7 @@
import logging
from paste.util.multidict import MultiDict
+from paste.deploy.converters import asbool
from ckan import model
log = logging.getLogger(__name__)
@@ -145,7 +146,7 @@
def validate(self):
for key, value in self.items():
if key in self.BOOLEAN_OPTIONS:
- value = value == 1 or value
+ value = asbool(value)
elif key in self.INTEGER_OPTIONS:
value = int(value)
self[key] = value
@@ -233,14 +234,13 @@
def validate(self):
""" Check that this is a valid query. """
- if not len(self.query):
- raise SearchError("No query has been specified")
+ pass
def __str__(self):
return self.query
def __repr__(self):
- return "Query(%s)" % self
+ return "Query(%r)" % self.query
class SearchIndex(object):
--- a/ckan/lib/search/sql.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/lib/search/sql.py Fri May 06 18:03:11 2011 +0100
@@ -294,3 +294,11 @@
results = self._run_sql(sql, [])
return [res[0] for res in results]
+ def get_index(self, pkg_ref):
+ pkg = model.Package.get(pkg_ref)
+ assert pkg
+ sql = "SELECT package_id, search_vector FROM package_search WHERE package_id = %s"
+ res = self.backend.connection.execute(sql, pkg.id)
+ search_vector = res.fetchall()
+ res.close()
+ return search_vector
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckan/tests/functional/api/test_misc.py Fri May 06 18:03:11 2011 +0100
@@ -0,0 +1,53 @@
+from paste.deploy.converters import asbool
+from ckan.tests.functional.api.base import *
+from ckan.lib.create_test_data import CreateTestData
+from ckan.tests import TestController as ControllerTestCase
+
+class MiscApiTestCase(ApiTestCase, ControllerTestCase):
+
+ @classmethod
+ def setup_class(self):
+ try:
+ CreateTestData.delete()
+ except:
+ pass
+ model.repo.init_db()
+ model.Session.remove()
+ CreateTestData.create()
+
+ @classmethod
+ def teardown_class(self):
+ model.Session.remove()
+ CreateTestData.delete()
+
+ # Todo: Move this method to the Model API?
+ def test_0_tag_counts(self):
+ offset = self.offset('/tag_counts')
+ res = self.app.get(offset, status=200)
+ assert '["russian", 2]' in res, res
+ assert '["tolstoy", 1]' in res, res
+
+class QosApiTestCase(ApiTestCase, ControllerTestCase):
+
+ def test_throughput(self):
+ if not asbool(config.get('ckan.enable_call_timing', "false")):
+ raise SkipTest
+ # Create some throughput.
+ import datetime
+ start = datetime.datetime.now()
+ offset = self.offset('/rest/package')
+ while datetime.datetime.now() - start < datetime.timedelta(0,10):
+ res = self.app.get(offset, status=[200])
+ # Check throughput.
+ offset = self.offset('/qos/throughput/')
+ res = self.app.get(offset, status=[200])
+ data = self.data_from_res(res)
+ throughput = float(data)
+ assert throughput > 1, throughput
+
+class TestMiscApi1(Api1TestCase, MiscApiTestCase): pass
+class TestQosApi1(Api1TestCase, QosApiTestCase): pass
+class TestMiscApi2(Api2TestCase, MiscApiTestCase): pass
+class TestQosApi2(Api2TestCase, QosApiTestCase): pass
+class TestMiscApiUnversioned(MiscApiTestCase, ApiUnversionedTestCase): pass
+class TestQosApiUnversioned(ApiUnversionedTestCase, QosApiTestCase): pass
--- a/ckan/tests/functional/api/test_model.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/tests/functional/api/test_model.py Fri May 06 18:03:11 2011 +0100
@@ -1,7 +1,5 @@
-from paste.deploy.converters import asbool
from ckan.tests.functional.api.base import *
from ckan.lib.create_test_data import CreateTestData
-from ckan.tests import is_search_supported
from ckan.tests import TestController as ControllerTestCase
class ModelApiTestCase(BaseModelApiTestCase):
@@ -517,395 +515,8 @@
assert rel_dict['object'] == object_ref, (rel_dict, object_ref)
assert rel_dict['type'] == type, (rel_dict, type)
assert rel_dict['comment'] == comment, (rel_dict, comment)
-
-
-class PackageSearchApiTestCase(ApiTestCase, ControllerTestCase):
-
- @classmethod
- def setup_class(self):
- if not is_search_supported():
- import nose
- raise nose.SkipTest
- indexer = TestSearchIndexer()
- CreateTestData.create()
- self.package_fixture_data = {
- 'name' : u'testpkg',
- 'title': 'Some Title',
- 'url': u'http://blahblahblah.mydomain',
- 'resources': [{u'url':u'http://blahblahblah.mydomain',
- u'format':u'', u'description':''}],
- 'tags': ['russion', 'novel'],
- 'license_id': u'gpl-3.0',
- 'extras': {'national_statistic':'yes',
- 'geographic_coverage':'England, Wales'},
- }
- CreateTestData.create_arbitrary(self.package_fixture_data)
- indexer.index()
- self.base_url = self.offset('/search/package')
-
- @classmethod
- def teardown_class(self):
- CreateTestData.delete()
-
- def test_01_uri_q(self):
- offset = self.base_url + '?q=%s' % self.package_fixture_data['name']
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'])
- assert res_dict['count'] == 1, res_dict['count']
-
- def assert_package_search_results(self, results, names=[u'testpkg']):
- for name in names:
- ref = self.package_ref_from_name(name)
- assert ref in results, (ref, results)
-
- def package_ref_from_name(self, package_name):
- package = self.get_package_by_name(package_name)
- return self.ref_package(package)
-
- def test_02_post_q(self):
- offset = self.base_url
- query = {'q':'testpkg'}
- res = self.app.post(offset, params=query, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'])
- assert res_dict['count'] == 1, res_dict['count']
-
- def test_03_uri_qjson(self):
- query = {'q': self.package_fixture_data['name']}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'])
- assert res_dict['count'] == 1, res_dict['count']
-
- def test_04_post_qjson(self):
- query = {'q': self.package_fixture_data['name']}
- json_query = self.dumps(query)
- offset = self.base_url
- res = self.app.post(offset, params=json_query, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'])
- assert res_dict['count'] == 1, res_dict['count']
-
- def test_05_uri_qjson_tags(self):
- query = {'q': 'annakarenina tags:russian tags:tolstoy'}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'], names=[u'annakarenina'])
- assert res_dict['count'] == 1, res_dict
-
- def test_05_uri_qjson_tags_multiple(self):
- query = {'q': 'tags:russian tags:tolstoy'}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'], names=[u'annakarenina'])
- assert res_dict['count'] == 1, res_dict
-
- def test_06_uri_q_tags(self):
- query = webhelpers.util.html_escape('annakarenina tags:russian tags:tolstoy')
- offset = self.base_url + '?q=%s' % query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'], names=[u'annakarenina'])
- assert res_dict['count'] == 1, res_dict['count']
-
- def test_07_uri_qjson_tags(self):
- query = {'q': '', 'tags':['tolstoy']}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'], names=[u'annakarenina'])
- assert res_dict['count'] == 1, res_dict
-
- def test_07_uri_qjson_tags_multiple(self):
- query = {'q': '', 'tags':['tolstoy', 'russian']}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'], names=[u'annakarenina'])
- assert res_dict['count'] == 1, res_dict
-
- def test_07_uri_qjson_tags_reverse(self):
- query = {'q': '', 'tags':['russian']}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'], names=[u'annakarenina'])
- assert res_dict['count'] == 2, res_dict
-
- def test_07_uri_qjson_extras(self):
- query = {"geographic_coverage":"England"}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'])
- assert res_dict['count'] == 1, res_dict
-
- def test_07_uri_qjson_extras_2(self):
- query = {"national_statistic":"yes"}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- self.assert_package_search_results(res_dict['results'])
- assert res_dict['count'] == 1, res_dict
-
-
- def test_08_all_fields(self):
- rating = model.Rating(user_ip_address=u'123.1.2.3',
- package=self.anna,
- rating=3.0)
- model.Session.add(rating)
- model.repo.commit_and_remove()
-
- query = {'q': 'russian', 'all_fields':1}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 2, res_dict
- for rec in res_dict['results']:
- if rec['name'] == 'annakarenina':
- anna_rec = rec
- break
- assert anna_rec['name'] == 'annakarenina', res_dict['results']
- assert anna_rec['title'] == 'A Novel By Tolstoy', anna_rec['title']
- assert anna_rec['license_id'] == u'other-open', anna_rec['license_id']
- assert len(anna_rec['tags']) == 2, anna_rec['tags']
- for expected_tag in ['russian', 'tolstoy']:
- assert expected_tag in anna_rec['tags']
- assert anna_rec['ratings_average'] == 3.0, anna_rec['ratings_average']
- assert anna_rec['ratings_count'] == 1, anna_rec['ratings_count']
-
- def test_09_just_tags(self):
- offset = self.base_url + '?tags=russian&all_fields=1'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 2, res_dict
-
- def test_10_multiple_tags_with_plus(self):
- offset = self.base_url + '?tags=tolstoy+russian&all_fields=1'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
- def test_10_multiple_tags_with_ampersand(self):
- offset = self.base_url + '?tags=tolstoy&tags=russian&all_fields=1'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
- def test_10_many_tags_with_ampersand(self):
- offset = self.base_url + '?tags=tolstoy&tags=russian&tags=tolstoy'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 1, res_dict
-
- def test_11_pagination_limit(self):
- offset = self.base_url + '?all_fields=1&tags=russian&limit=1&order_by=name'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 2, res_dict
- assert len(res_dict['results']) == 1, res_dict
- assert res_dict['results'][0]['name'] == 'annakarenina', res_dict['results'][0]['name']
-
- def test_11_pagination_offset_limit(self):
- offset = self.base_url + '?all_fields=1&tags=russian&offset=1&limit=1&order_by=name'
- res = self.app.get(offset, status=200)
- res_dict = self.data_from_res(res)
- assert res_dict['count'] == 2, res_dict
- assert len(res_dict['results']) == 1, res_dict
- assert res_dict['results'][0]['name'] == 'warandpeace', res_dict['results'][0]['name']
-
- def test_12_search_revision_basic(self):
- offset = self.offset('/search/revision')
- # Check bad request.
- self.app.get(offset, status=400)
- self.app.get(offset+'?since_rev=2010-01-01T00:00:00', status=400)
- self.app.get(offset+'?since_revision=2010-01-01T00:00:00', status=400)
- self.app.get(offset+'?since_id=', status=400)
-
- def test_12_search_revision_since_rev(self):
- offset = self.offset('/search/revision')
- revs = model.Session.query(model.Revision).all()
- rev_first = revs[-1]
- params = "?since_id=%s" % str(rev_first.id)
- res = self.app.get(offset+params, status=200)
- res_list = self.data_from_res(res)
- assert rev_first.id not in res_list
- for rev in revs[:-1]:
- assert rev.id in res_list, (rev.id, res_list)
- rev_last = revs[0]
- params = "?since_id=%s" % str(rev_last.id)
- res = self.app.get(offset+params, status=200)
- res_list = self.data_from_res(res)
- assert res_list == [], res_list
-
- def test_12_search_revision_since_time(self):
- offset = self.offset('/search/revision')
- revs = model.Session.query(model.Revision).all()
- # Check since time of first.
- rev_first = revs[-1]
- params = "?since_time=%s" % model.strftimestamp(rev_first.timestamp)
- res = self.app.get(offset+params, status=200)
- res_list = self.data_from_res(res)
- assert rev_first.id not in res_list
- for rev in revs[:-1]:
- assert rev.id in res_list, (rev.id, res_list)
- # Check since time of last.
- rev_last = revs[0]
- params = "?since_time=%s" % model.strftimestamp(rev_last.timestamp)
- res = self.app.get(offset+params, status=200)
- res_list = self.data_from_res(res)
- assert res_list == [], res_list
- # Check bad format.
- params = "?since_time=2010-04-31T23:45"
- self.app.get(offset+params, status=400)
-
- def test_strftimestamp(self):
- import datetime
- t = datetime.datetime(2012, 3, 4, 5, 6, 7, 890123)
- s = model.strftimestamp(t)
- assert s == "2012-03-04T05:06:07.890123", s
-
- def test_strptimestamp(self):
- import datetime
- s = "2012-03-04T05:06:07.890123"
- t = model.strptimestamp(s)
- assert t == datetime.datetime(2012, 3, 4, 5, 6, 7, 890123), t
-
-
-class ResourceSearchApiTestCase(ApiTestCase, ControllerTestCase):
-
- @classmethod
- def setup_class(self):
- CreateTestData.create()
- self.ab = 'http://site.com/a/b.txt'
- self.cd = 'http://site.com/c/d.txt'
- self.package_fixture_data = {
- 'name' : u'testpkg',
- 'title': 'Some Title',
- 'url': u'http://blahblahblah.mydomain',
- 'resources':[
- {'url':self.ab,
- 'description':'This is site ab.',
- 'format':'Excel spreadsheet',
- 'alt_url':'alt',
- 'extras':{'size':'100'},
- 'hash':'abc-123'},
- {'url':self.cd,
- 'description':'This is site cd.',
- 'format':'Office spreadsheet',
- 'alt_url':'alt',
- 'extras':{'size':'100'},
- 'hash':'qwe-456'},
- ],
- 'tags': ['russion', 'novel'],
- 'license_id': u'gpl-3.0',
- 'extras': {'national_statistic':'yes',
- 'geographic_coverage':'England, Wales'},
- }
- CreateTestData.create_arbitrary(self.package_fixture_data)
- self.base_url = self.offset('/search/resource')
-
- @classmethod
- def teardown_class(self):
- CreateTestData.delete()
-
- def assert_urls_in_search_results(self, offset, expected_urls):
- result = self.app.get(offset, status=200)
- result_dict = self.loads(result.body)
- resources = [model.Session.query(model.Resource).get(resource_id) for resource_id in result_dict['results']]
- urls = set([resource.url for resource in resources])
- assert urls == set(expected_urls), urls
-
-
- def test_01_url(self):
- offset = self.base_url + '?url=site'
- self.assert_urls_in_search_results(offset, [self.ab, self.cd])
-
- def test_02_url_qjson(self):
- query = {'url':'site'}
- json_query = self.dumps(query)
- offset = self.base_url + '?qjson=%s' % json_query
- self.assert_urls_in_search_results(offset, [self.ab, self.cd])
-
- def test_03_post_qjson(self):
- query = {'url':'site'}
- json_query = self.dumps(query)
- offset = self.base_url
- result = self.app.post(offset, params=json_query, status=200)
- expected_urls = [self.ab, self.cd]
- result_dict = self.loads(result.body)
- resources = [model.Session.query(model.Resource).get(resource_id) for resource_id in result_dict['results']]
- urls = set([resource.url for resource in resources])
- assert urls == set(expected_urls), urls
-
- def test_04_bad_option(self):
- offset = self.base_url + '?random=option'
- result = self.app.get(offset, status=400)
-
- def test_05_options(self):
- offset = self.base_url + '?url=site&all_fields=1&callback=mycallback'
- result = self.app.get(offset, status=200)
- assert re.match('^mycallback\(.*\);$', result.body), result.body
- assert 'package_id' in result.body, result.body
-
-
-class QosApiTestCase(ApiTestCase, ControllerTestCase):
-
- def test_throughput(self):
- if not asbool(config.get('ckan.enable_call_timing', "false")):
- raise SkipTest
- # Create some throughput.
- import datetime
- start = datetime.datetime.now()
- offset = self.offset('/rest/package')
- while datetime.datetime.now() - start < datetime.timedelta(0,10):
- res = self.app.get(offset, status=[200])
- # Check throughput.
- offset = self.offset('/qos/throughput/')
- res = self.app.get(offset, status=[200])
- data = self.data_from_res(res)
- throughput = float(data)
- assert throughput > 1, throughput
-class MiscApiTestCase(ApiTestCase, ControllerTestCase):
-
- @classmethod
- def setup_class(self):
- try:
- CreateTestData.delete()
- except:
- pass
- model.repo.init_db()
- model.Session.remove()
- CreateTestData.create()
-
- @classmethod
- def teardown_class(self):
- model.Session.remove()
- CreateTestData.delete()
-
- # Todo: Move this method to the Model API?
- def test_0_tag_counts(self):
- offset = self.offset('/tag_counts')
- res = self.app.get(offset, status=200)
- assert '["russian", 2]' in res, res
- assert '["tolstoy", 1]' in res, res
-
-
# Tests for Version 1 of the API.
class TestModelApi1(Api1TestCase, ModelApiTestCase):
@@ -956,34 +567,12 @@
class TestRelationshipsApi1(Api1TestCase, RelationshipsApiTestCase): pass
-class TestPackageSearchApi1(Api1TestCase, PackageSearchApiTestCase): pass
-class TestResourceSearchApi1(ResourceSearchApiTestCase, Api1TestCase): pass
-class TestMiscApi1(Api1TestCase, MiscApiTestCase): pass
-class TestQosApi1(Api1TestCase, QosApiTestCase): pass
# Tests for Version 2 of the API.
class TestModelApi2(Api2TestCase, ModelApiTestCase): pass
class TestRelationshipsApi2(Api2TestCase, RelationshipsApiTestCase): pass
-class TestPackageSearchApi2(Api2TestCase, PackageSearchApiTestCase): pass
-class TestResourceSearchApi2(Api2TestCase, ResourceSearchApiTestCase): pass
-class TestMiscApi2(Api2TestCase, MiscApiTestCase): pass
-class TestQosApi2(Api2TestCase, QosApiTestCase): pass
# Tests for unversioned API.
class TestModelApiUnversioned(ApiUnversionedTestCase, ModelApiTestCase): pass
+class TestRelationshipsApiUnversioned(RelationshipsApiTestCase, ApiUnversionedTestCase): pass
-# Todo: Refactor to run the download_url tests on versioned location too.
-#class TestRelationshipsApiUnversioned(RelationshipsApiTestCase, ApiUnversionedTestCase):
-# pass
-#
-#class TestPackageSearchApiUnversioned(PackageSearchApiTestCase, ApiUnversionedTestCase):
-# pass
-#
-class TestResourceSearchApiUnversioned(ApiUnversionedTestCase, ResourceSearchApiTestCase):
- pass
-
-#class TestMiscApiUnversioned(MiscApiTestCase, ApiUnversionedTestCase):
-# pass
-
-class TestQosApiUnversioned(ApiUnversionedTestCase, QosApiTestCase): pass
-
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckan/tests/functional/api/test_package_search.py Fri May 06 18:03:11 2011 +0100
@@ -0,0 +1,316 @@
+from nose.tools import assert_raises
+
+from ckan.tests import is_search_supported
+from ckan.tests.functional.api.base import *
+from ckan.tests import TestController as ControllerTestCase
+from ckan.controllers.api import ApiController
+from webob.multidict import UnicodeMultiDict
+
+class PackageSearchApiTestCase(ApiTestCase, ControllerTestCase):
+
+ @classmethod
+ def setup_class(self):
+ if not is_search_supported():
+ import nose
+ raise nose.SkipTest
+ indexer = TestSearchIndexer()
+ CreateTestData.create()
+ self.package_fixture_data = {
+ 'name' : u'testpkg',
+ 'title': 'Some Title',
+ 'url': u'http://blahblahblah.mydomain',
+ 'resources': [{u'url':u'http://blahblahblah.mydomain',
+ u'format':u'', u'description':''}],
+ 'tags': ['russion', 'novel'],
+ 'license_id': u'gpl-3.0',
+ 'extras': {'national_statistic':'yes',
+ 'geographic_coverage':'England, Wales'},
+ }
+ CreateTestData.create_arbitrary(self.package_fixture_data)
+ self.base_url = self.offset('/search/package')
+
+ @classmethod
+ def teardown_class(self):
+ CreateTestData.delete()
+
+ def assert_results(self, res_dict, expected_package_names):
+ expected_pkgs = [self.package_ref_from_name(expected_package_name) \
+ for expected_package_name in expected_package_names]
+ assert_equal(set(res_dict['results']), set(expected_pkgs))
+
+ def package_ref_from_name(self, package_name):
+ package = self.get_package_by_name(package_name)
+ return self.ref_package(package)
+
+ def test_00_read_search_params(self):
+ def check(request_params, expected_params):
+ params = ApiController._get_search_params(request_params)
+ assert_equal(params, expected_params)
+ # uri parameters
+ check(UnicodeMultiDict({'q': '', 'ref': 'boris'}),
+ {"q": "", "ref": "boris"})
+ check(UnicodeMultiDict({'filter_by_openness': '1'}),
+ {'filter_by_openness': '1'})
+ # uri json
+ check(UnicodeMultiDict({'qjson': '{"q": "", "ref": "boris"}'}),
+ {"q": "", "ref": "boris"})
+ # posted json
+ check(UnicodeMultiDict({'{"q": "", "ref": "boris"}': u'1'}),
+ {"q": "", "ref": "boris"})
+ check(UnicodeMultiDict({'{"q": "", "ref": "boris"}': u''}),
+ {"q": "", "ref": "boris"})
+ # no parameters
+ check(UnicodeMultiDict({}),
+ {})
+
+ def test_00_read_search_params_with_errors(self):
+ def check_error(request_params):
+ assert_raises(ValueError, ApiController._get_search_params, request_params)
+ # uri json
+ check_error(UnicodeMultiDict({'qjson': '{"q": illegal json}'}))
+ # posted json
+ check_error(UnicodeMultiDict({'{"q": illegal json}': u'1'}))
+
+ def test_01_uri_q(self):
+ offset = self.base_url + '?q=%s' % self.package_fixture_data['name']
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict['count']
+
+ def test_02_post_q(self):
+ offset = self.base_url
+ query = {'q':'testpkg'}
+ res = self.app.post(offset, params=query, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict['count']
+
+ def test_03_uri_qjson(self):
+ query = {'q': self.package_fixture_data['name']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict['count']
+
+ def test_04_post_qjson(self):
+ query = {'q': self.package_fixture_data['name']}
+ json_query = self.dumps(query)
+ offset = self.base_url
+ res = self.app.post(offset, params=json_query, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict['count']
+
+ def test_05_uri_qjson_tags(self):
+ query = {'q': 'annakarenina tags:russian tags:tolstoy'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_05_uri_qjson_tags_multiple(self):
+ query = {'q': 'tags:russian tags:tolstoy'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_06_uri_q_tags(self):
+ query = webhelpers.util.html_escape('annakarenina tags:russian tags:tolstoy')
+ offset = self.base_url + '?q=%s' % query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict['count']
+
+ def test_07_uri_qjson_tags(self):
+ query = {'q': '', 'tags':['tolstoy']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_tags_multiple(self):
+ query = {'q': '', 'tags':['tolstoy', 'russian']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ print offset
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_tags_reverse(self):
+ query = {'q': '', 'tags':['russian']}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, [u'annakarenina', u'warandpeace'])
+ assert res_dict['count'] == 2, res_dict
+
+ def test_07_uri_qjson_extras(self):
+ query = {"geographic_coverage":"England"}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_07_uri_qjson_extras_2(self):
+ query = {"national_statistic":"yes"}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ self.assert_results(res_dict, ['testpkg'])
+ assert res_dict['count'] == 1, res_dict
+
+ def test_08_uri_qjson_malformed(self):
+ offset = self.base_url + '?qjson="q":""' # user forgot the curly braces
+ res = self.app.get(offset, status=400)
+
+ def test_08_all_fields(self):
+ rating = model.Rating(user_ip_address=u'123.1.2.3',
+ package=self.anna,
+ rating=3.0)
+ model.Session.add(rating)
+ model.repo.commit_and_remove()
+
+ query = {'q': 'russian', 'all_fields':1}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ for rec in res_dict['results']:
+ if rec['name'] == 'annakarenina':
+ anna_rec = rec
+ break
+ assert anna_rec['name'] == 'annakarenina', res_dict['results']
+ assert anna_rec['title'] == 'A Novel By Tolstoy', anna_rec['title']
+ assert anna_rec['license_id'] == u'other-open', anna_rec['license_id']
+ assert len(anna_rec['tags']) == 2, anna_rec['tags']
+ for expected_tag in ['russian', 'tolstoy']:
+ assert expected_tag in anna_rec['tags']
+ assert anna_rec['ratings_average'] == 3.0, anna_rec['ratings_average']
+ assert anna_rec['ratings_count'] == 1, anna_rec['ratings_count']
+
+ def test_09_just_tags(self):
+ offset = self.base_url + '?tags=russian&all_fields=1'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+
+ def test_10_multiple_tags_with_plus(self):
+ offset = self.base_url + '?tags=tolstoy+russian&all_fields=1'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_10_multiple_tags_with_ampersand(self):
+ offset = self.base_url + '?tags=tolstoy&tags=russian&all_fields=1'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_10_many_tags_with_ampersand(self):
+ offset = self.base_url + '?tags=tolstoy&tags=russian&tags=tolstoy'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 1, res_dict
+
+ def test_11_pagination_limit(self):
+ offset = self.base_url + '?all_fields=1&tags=russian&limit=1&order_by=name'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ assert len(res_dict['results']) == 1, res_dict
+ assert res_dict['results'][0]['name'] == 'annakarenina', res_dict['results'][0]['name']
+
+ def test_11_pagination_offset_limit(self):
+ offset = self.base_url + '?all_fields=1&tags=russian&offset=1&limit=1&order_by=name'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert res_dict['count'] == 2, res_dict
+ assert len(res_dict['results']) == 1, res_dict
+ assert res_dict['results'][0]['name'] == 'warandpeace', res_dict['results'][0]['name']
+
+ def test_12_all_packages_qjson(self):
+ query = {'q': ''}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 3)
+
+ def test_12_all_packages_q(self):
+ offset = self.base_url + '?q=""'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 3)
+
+ def test_12_all_packages_no_q(self):
+ offset = self.base_url
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 3)
+
+ def test_12_filter_by_openness_qjson(self):
+ query = {'q': '', 'filter_by_openness': '1'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 2)
+ self.assert_results(res_dict, (u'annakarenina', u'testpkg'))
+
+ def test_12_filter_by_openness_q(self):
+ offset = self.base_url + '?filter_by_openness=1'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 2)
+ self.assert_results(res_dict, (u'annakarenina', u'testpkg'))
+
+ def test_12_filter_by_openness_off_qjson(self):
+ query = {'q': '', 'filter_by_openness': '0'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 3)
+
+ def test_12_filter_by_openness_off_q(self):
+ offset = self.base_url + '?filter_by_openness=0'
+ res = self.app.get(offset, status=200)
+ res_dict = self.data_from_res(res)
+ assert_equal(res_dict['count'], 3)
+
+ def test_strftimestamp(self):
+ import datetime
+ t = datetime.datetime(2012, 3, 4, 5, 6, 7, 890123)
+ s = model.strftimestamp(t)
+ assert s == "2012-03-04T05:06:07.890123", s
+
+ def test_strptimestamp(self):
+ import datetime
+ s = "2012-03-04T05:06:07.890123"
+ t = model.strptimestamp(s)
+ assert t == datetime.datetime(2012, 3, 4, 5, 6, 7, 890123), t
+
+class TestPackageSearchApi1(Api1TestCase, PackageSearchApiTestCase): pass
+class TestPackageSearchApi2(Api2TestCase, PackageSearchApiTestCase): pass
+class TestPackageSearchApiUnversioned(PackageSearchApiTestCase, ApiUnversionedTestCase): pass
+
+
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckan/tests/functional/api/test_resource_search.py Fri May 06 18:03:11 2011 +0100
@@ -0,0 +1,84 @@
+from ckan.tests.functional.api.base import *
+from ckan.tests import TestController as ControllerTestCase
+
+class ResourceSearchApiTestCase(ApiTestCase, ControllerTestCase):
+
+ @classmethod
+ def setup_class(self):
+ CreateTestData.create()
+ self.ab = 'http://site.com/a/b.txt'
+ self.cd = 'http://site.com/c/d.txt'
+ self.package_fixture_data = {
+ 'name' : u'testpkg',
+ 'title': 'Some Title',
+ 'url': u'http://blahblahblah.mydomain',
+ 'resources':[
+ {'url':self.ab,
+ 'description':'This is site ab.',
+ 'format':'Excel spreadsheet',
+ 'alt_url':'alt',
+ 'extras':{'size':'100'},
+ 'hash':'abc-123'},
+ {'url':self.cd,
+ 'description':'This is site cd.',
+ 'format':'Office spreadsheet',
+ 'alt_url':'alt',
+ 'extras':{'size':'100'},
+ 'hash':'qwe-456'},
+ ],
+ 'tags': ['russion', 'novel'],
+ 'license_id': u'gpl-3.0',
+ 'extras': {'national_statistic':'yes',
+ 'geographic_coverage':'England, Wales'},
+ }
+ CreateTestData.create_arbitrary(self.package_fixture_data)
+ self.base_url = self.offset('/search/resource')
+
+ @classmethod
+ def teardown_class(self):
+ CreateTestData.delete()
+
+ def assert_urls_in_search_results(self, offset, expected_urls):
+ result = self.app.get(offset, status=200)
+ result_dict = self.loads(result.body)
+ resources = [model.Session.query(model.Resource).get(resource_id) for resource_id in result_dict['results']]
+ urls = set([resource.url for resource in resources])
+ assert urls == set(expected_urls), urls
+
+
+ def test_01_url(self):
+ offset = self.base_url + '?url=site'
+ self.assert_urls_in_search_results(offset, [self.ab, self.cd])
+
+ def test_02_url_qjson(self):
+ query = {'url':'site'}
+ json_query = self.dumps(query)
+ offset = self.base_url + '?qjson=%s' % json_query
+ self.assert_urls_in_search_results(offset, [self.ab, self.cd])
+
+ def test_03_post_qjson(self):
+ query = {'url':'site'}
+ json_query = self.dumps(query)
+ offset = self.base_url
+ result = self.app.post(offset, params=json_query, status=200)
+ expected_urls = [self.ab, self.cd]
+ result_dict = self.loads(result.body)
+ resources = [model.Session.query(model.Resource).get(resource_id) for resource_id in result_dict['results']]
+ urls = set([resource.url for resource in resources])
+ assert urls == set(expected_urls), urls
+
+ def test_04_bad_option(self):
+ offset = self.base_url + '?random=option'
+ result = self.app.get(offset, status=400)
+
+ def test_05_options(self):
+ offset = self.base_url + '?url=site&all_fields=1&callback=mycallback'
+ result = self.app.get(offset, status=200)
+ assert re.match('^mycallback\(.*\);$', result.body), result.body
+ assert 'package_id' in result.body, result.body
+
+
+class TestResourceSearchApi1(ResourceSearchApiTestCase, Api1TestCase): pass
+class TestResourceSearchApi2(Api2TestCase, ResourceSearchApiTestCase): pass
+class TestResourceSearchApiUnversioned(ApiUnversionedTestCase, ResourceSearchApiTestCase):
+ pass
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckan/tests/functional/api/test_revision_search.py Fri May 06 18:03:11 2011 +0100
@@ -0,0 +1,61 @@
+from ckan.tests.functional.api.base import *
+from ckan.tests import TestController as ControllerTestCase
+
+class RevisionSearchApiTestCase(ApiTestCase, ControllerTestCase):
+
+ @classmethod
+ def setup_class(self):
+ CreateTestData.create()
+
+ @classmethod
+ def teardown_class(self):
+ CreateTestData.delete()
+
+ def test_12_search_revision_basic(self):
+ offset = self.offset('/search/revision')
+ # Check bad request.
+ self.app.get(offset, status=400)
+ self.app.get(offset+'?since_rev=2010-01-01T00:00:00', status=400)
+ self.app.get(offset+'?since_revision=2010-01-01T00:00:00', status=400)
+ self.app.get(offset+'?since_id=', status=400)
+
+ def test_12_search_revision_since_rev(self):
+ offset = self.offset('/search/revision')
+ revs = model.Session.query(model.Revision).all()
+ rev_first = revs[-1]
+ params = "?since_id=%s" % str(rev_first.id)
+ res = self.app.get(offset+params, status=200)
+ res_list = self.data_from_res(res)
+ assert rev_first.id not in res_list
+ for rev in revs[:-1]:
+ assert rev.id in res_list, (rev.id, res_list)
+ rev_last = revs[0]
+ params = "?since_id=%s" % str(rev_last.id)
+ res = self.app.get(offset+params, status=200)
+ res_list = self.data_from_res(res)
+ assert res_list == [], res_list
+
+ def test_12_search_revision_since_time(self):
+ offset = self.offset('/search/revision')
+ revs = model.Session.query(model.Revision).all()
+ # Check since time of first.
+ rev_first = revs[-1]
+ params = "?since_time=%s" % model.strftimestamp(rev_first.timestamp)
+ res = self.app.get(offset+params, status=200)
+ res_list = self.data_from_res(res)
+ assert rev_first.id not in res_list
+ for rev in revs[:-1]:
+ assert rev.id in res_list, (rev.id, res_list)
+ # Check since time of last.
+ rev_last = revs[0]
+ params = "?since_time=%s" % model.strftimestamp(rev_last.timestamp)
+ res = self.app.get(offset+params, status=200)
+ res_list = self.data_from_res(res)
+ assert res_list == [], res_list
+ # Check bad format.
+ params = "?since_time=2010-04-31T23:45"
+ self.app.get(offset+params, status=400)
+
+
+class TestPackageSearchApi1(Api1TestCase, RevisionSearchApiTestCase): pass
+class TestPackageSearchApi2(Api2TestCase, RevisionSearchApiTestCase): pass
--- a/ckan/tests/functional/test_authorization_group.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/tests/functional/test_authorization_group.py Fri May 06 18:03:11 2011 +0100
@@ -125,3 +125,269 @@
# now look at packages
assert len(group.users) == 2
+
+
+class TestAuthorizationGroupWalkthrough(FunctionalTestCase):
+
+ @classmethod
+ def setup_class(self):
+ model.Session.remove()
+ model.repo.init_db()
+ CreateTestData.create()
+ model.repo.commit_and_remove()
+
+
+ @classmethod
+ def teardown_class(self):
+ model.Session.remove()
+ model.repo.rebuild_db()
+ model.Session.remove()
+
+ def test_authzgroups_walkthrough(self):
+ # very long test sequence repeating the series of things I did to
+ # convince myself that the authzgroups system worked as expected,
+ # starting off with the default test data
+
+ # The first thing to notice is that the authzgroup page:
+ auth_group_index_url = url_for(controller='/authorization_group', action='index')
+ # displays differently for different users.
+
+ def get_page(url, expect_status, username, assert_text=None, error_text=None):
+ res= self.app.get(url,
+ status=expect_status,
+ extra_environ={'REMOTE_USER': username})
+ if assert_text and assert_text not in res:
+ errorstring = error_text + ' ( "' + assert_text + \
+ '" not found in result of getting "' + \
+ url + '" as user "' + username + '" )'
+ assert False, errorstring
+ return res
+
+ # testsysadmin sees the true picture, where the test data contains two groups
+ get_page(auth_group_index_url, 200, 'testsysadmin',
+ 'There are <strong>2</strong> authorization groups',
+ 'Should be accurate for testsysadmin')
+
+ # But if we look at the same page as annafan, who does not have read
+ # permissions on these groups, we should see neither
+ get_page(auth_group_index_url, 200, 'annafan',
+ 'There are <strong>0</strong> authorization groups',
+ 'Should lie to annafan about number of groups')
+
+ # There is a page for each group
+ anauthzgroup_url = url_for(controller='/authorization_group',
+ action='read',
+ id='anauthzgroup')
+ # And an edit page
+ anauthzgroup_edit_url = url_for(controller='/authorization_group',
+ action='edit',
+ id='anauthzgroup')
+
+ # testsysadmin should be able to see this, and check that there are no members
+ get_page(anauthzgroup_url, 200, 'testsysadmin',
+ 'There are 0 users in this',
+ 'should be no users in anauthzgroup')
+
+ # now testsysadmin adds annafan to anauthzgroup via the edit page
+ res = get_page(anauthzgroup_edit_url, 200, 'testsysadmin')
+ group_edit_form = res.forms['group-edit']
+ group_edit_form['AuthorizationGroupUser--user_name'] = u'annafan'
+ submit_res = group_edit_form.submit('save',
+ extra_environ={'REMOTE_USER': 'testsysadmin'})
+
+ # adding a user to a group should both make her a member, and give her
+ # read permission on the group. We'll check those things have actually
+ # happened by looking directly in the model.
+ anauthzgroup = model.AuthorizationGroup.by_name('anauthzgroup')
+ anauthzgroup_users = [x.name for x in anauthzgroup.users]
+ anauthzgroup_user_roles = [(x.user.name, x.role) for x in anauthzgroup.roles if x.user]
+ assert anauthzgroup_users == [u'annafan'], \
+ 'anauthzgroup should contain annafan (only)'
+ assert anauthzgroup_user_roles == [(u'annafan', u'reader')],\
+ 'annafan should be a reader'
+
+ # Since annafan has been added to anauthzgroup, which is an admin on
+ # anotherauthzgroup, she should now be able to see both the groups.
+ get_page(auth_group_index_url, 200, 'annafan',
+ 'There are <strong>2</strong> auth',
+ "annafan should now be able to see both groups")
+
+ # When annafan looks at the page for anauthzgroup now
+ # She should see that there's one user:
+ get_page(anauthzgroup_url, 200,'annafan',
+ 'There are 1 users in this',
+ 'annafan should be able to see the list of members')
+
+ # Which is her, so her name should be in there somewhere:
+ get_page(anauthzgroup_url, 200,'annafan',
+ 'annafan',
+ 'annafan should be listed as a member')
+
+ # But she shouldn't be able to see the edit page for that group.
+
+ # The behaviour of the test setup here is a bit weird, since in the
+ # browser she gets redirected to the login page, but from these tests,
+ # she just gets a 401, with no apparent redirect. Sources inform me
+ # that this is normal, and to do with repoze being in the application
+ # stack but not in the test stack.
+ get_page(anauthzgroup_edit_url, 401, 'annafan',
+ 'not authorized to edit',
+ 'annafan should not be able to edit the list of members')
+ # this behaviour also means that we get a flash message left over, which appears on
+ # whatever the next page is.
+
+ # I'm going to assert that behaviour here, just to note it. It's most
+ # definitely not required functionality! We'll do a dummy fetch of the
+ # main page for anauthzgroup, which will have the errant flash message
+ get_page(anauthzgroup_url, 200, 'annafan',
+ 'not authorized to edit',
+ 'flash message should carry over to next fetch')
+
+ # But if we do the dummy fetch twice, the flash message should have gone
+ res = get_page(anauthzgroup_url, 200, 'annafan')
+ assert 'not authorized to edit' not in res, 'flash message should have gone'
+
+ # Since annafan is now a member of anauthzgroup, she should have admin privileges
+ # on anotherauthzgroup
+ anotherauthzgroup_edit_url = url_for(controller='/authorization_group',
+ action='edit',
+ id='anotherauthzgroup')
+
+ # Which means that she can go to the edit page:
+ res = get_page(anotherauthzgroup_edit_url, 200, 'annafan',
+ 'There are no users',
+ "There shouldn't be any users in anotherauthzgroup")
+
+ # And change the name of the group
+ # The group name editing box has a name dependent on the id of the group,
+ # so we find it by regex in the page.
+ import re
+ p = re.compile('AuthorizationGroup-.*-name')
+ groupnamebox = [ v for k,v in res.forms['group-edit'].fields.items() if p.match(k)][0][0]
+ groupnamebox.value = 'annasauthzgroup'
+ res = res.forms['group-edit'].submit('save', extra_environ={'REMOTE_USER': 'annafan'})
+ res = res.follow()
+
+ ## POTENTIAL BUG:
+ # note that she could change the name of the group to anauthzgroup,
+ # which causes problems due to the name collision. This should be
+ # guarded against.
+
+
+ # annafan should still be able to see the admin and edit pages of the
+ # newly renamed group by virtue of being a member of anauthzgroup
+ annasauthzgroup_authz_url = url_for(controller='/authorization_group',
+ action='authz',
+ id='annasauthzgroup')
+
+ annasauthzgroup_edit_url = url_for(controller='/authorization_group',
+ action='edit',
+ id='annasauthzgroup')
+
+
+ res = get_page(annasauthzgroup_authz_url, 200, 'annafan',
+ 'Authorization for authorization group: annasauthzgroup',
+ 'should be authz page')
+
+ # annafan has the power to remove anauthzgroup's admin role on her group
+ # The button to remove that role is a link, rather than a submit. I
+ # assume there is a better way to do this than searching by regex, but I
+ # can't find it.
+ import re
+ delete_links = re.compile('<a href="(.*)" title="delete">').findall(res.body)
+ assert len(delete_links) == 1, "There should only be one delete link here"
+ delete_link = delete_links[0]
+
+ # Paranoid check, try to follow link without credentials. Should be redirected.
+ res = self.app.get(delete_link, status=302)
+ res = res.follow()
+ assert 'Not authorized to edit authorization for group' in res,\
+ "following link without credentials should result in redirection to login page"
+
+ # Now follow it as annafan, which should work.
+ get_page(delete_link, 200,'annafan',
+ "Deleted role 'admin' for authorization group 'anauthzgroup'",
+ "Page should mention the deleted role")
+
+ # Trying it a second time should fail since she's now not an admin.
+ get_page(delete_link, 401,'annafan')
+
+ # No one should now have any rights on annasauthzgroup, including
+ # annafan herself. So this should fail too. Again, get a 401 error
+ # here, but in the browser we get redirected if we try.
+ get_page(annasauthzgroup_authz_url, 401,'annafan')
+
+ # testsysadmin can put her back.
+ # It appears that the select boxes on this form need to be set by id
+ anauthzgroupid = model.AuthorizationGroup.by_name(u'anauthzgroup').id
+ annafanid = model.User.by_name(u'annafan').id
+
+ # first try to make both anauthzgroup and annafan editors. This should fail.
+ res = get_page(annasauthzgroup_authz_url,200, 'testsysadmin')
+ gaf= res.forms['group-authz']
+ gaf['AuthorizationGroupRole--authorized_group_id'] = anauthzgroupid
+ gaf['AuthorizationGroupRole--role'] = 'editor'
+ gaf['AuthorizationGroupRole--user_id'] = annafanid
+ res = gaf.submit('save', status=200, extra_environ={'REMOTE_USER': 'testsysadmin'})
+ assert 'Please select either a user or an authorization group, not both.' in res,\
+ 'request should fail if you change both user and authz group'
+
+ # settle for just doing one at a time. make anauthzgroup an editor
+ res = get_page(annasauthzgroup_authz_url, 200, 'testsysadmin')
+ gaf= res.forms['group-authz']
+ gaf['AuthorizationGroupRole--authorized_group_id'] = anauthzgroupid
+ gaf['AuthorizationGroupRole--role'] = 'editor'
+ res = gaf.submit('save',status=200, extra_environ={'REMOTE_USER': 'testsysadmin'})
+ assert "Added role 'editor' for authorization group 'anauthzgroup'" in res, \
+ "no flash message"
+
+ # and make annafan a reader
+ res = get_page(annasauthzgroup_authz_url, 200, 'testsysadmin')
+ gaf= res.forms['group-authz']
+ gaf['AuthorizationGroupRole--user_id'] = annafanid
+ gaf['AuthorizationGroupRole--role'] = 'reader'
+ res = gaf.submit('save', status=200, extra_environ={'REMOTE_USER': 'testsysadmin'})
+ assert "Added role 'reader' for user 'annafan'" in res, "no flash message"
+
+ # annafan should now be able to add her friends to annasauthzgroup
+ res = get_page(annasauthzgroup_edit_url, 200, 'annafan')
+ res.forms['group-edit']['AuthorizationGroupUser--user_name']='tester'
+ # this follows the post/redirect/get pattern
+ res = res.forms['group-edit'].submit('save', status=302,
+ extra_environ={'REMOTE_USER': 'annafan'})
+ res = res.follow(status=200, extra_environ={'REMOTE_USER': 'annafan'})
+ # and she gets redirected to the group view page
+ assert 'tester' in res, 'tester not added?'
+
+ # she needs to do them one by one
+ res = get_page(annasauthzgroup_edit_url, 200, 'annafan',
+ 'tester',
+ 'tester not in edit form')
+ res.forms['group-edit']['AuthorizationGroupUser--user_name']='russianfan'
+ res = res.forms['group-edit'].submit('save', status=302, extra_environ={'REMOTE_USER': 'annafan'})
+ res = res.follow(status=200, extra_environ={'REMOTE_USER': 'annafan'})
+
+ # and finally adds herself
+ res = self.app.get(annasauthzgroup_edit_url, status=200, extra_environ={'REMOTE_USER': 'annafan'})
+ assert 'russianfan' in res, 'russianfan not added?'
+ res.forms['group-edit']['AuthorizationGroupUser--user_name']='annafan'
+ res = res.forms['group-edit'].submit('save', status=302, extra_environ={'REMOTE_USER': 'annafan'})
+ res = res.follow(status=200, extra_environ={'REMOTE_USER': 'annafan'})
+ assert 'annafan' in res, 'annafan not added?'
+
+ # finally let's check that annafan can create a completely new authzgroup
+ new_authzgroup_url = url_for(controller='/authorization_group', action='new')
+ res = get_page(new_authzgroup_url, 200,'annafan',
+ 'New Authorization Group',
+ "wrong page?")
+ gef = res.forms['group-edit']
+ gef['AuthorizationGroup--name']="newgroup"
+ gef['AuthorizationGroupUser--user_name'] = "russianfan"
+ res = gef.submit('save', status=302, extra_environ={'REMOTE_USER': 'annafan'})
+ #post/redirect/get
+ res = res.follow(status=200, extra_environ={'REMOTE_USER': 'annafan'})
+
+ assert 'newgroup' in res, "should have redirected to the newgroup page"
+ assert 'russianfan' in res, "no russianfan"
+ assert 'There are 1 users in this authorization group' in res, "missing text"
+
--- a/ckan/tests/functional/test_revision.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/tests/functional/test_revision.py Fri May 06 18:03:11 2011 +0100
@@ -84,7 +84,7 @@
try:
# paginate links are also just numbers
# res2 = res.click('^%s$' % link_exp)
- res2 = res.click(href='revision/read/%s$' % link_exp)
+ res2 = res.click(link_exp)
except:
print "\nThe first response (list):\n\n"
print str(res)
@@ -142,11 +142,6 @@
def get_package(self, name):
return model.Package.by_name(name)
- def test_read_redirect_at_base(self):
- # have to put None as o/w seems to still be at url set in previous test
- offset = url_for(controller='revision', action='read', id=None)
- res = self.app.get(offset, status=404)
-
def test_read(self):
anna = model.Package.by_name(u'annakarenina')
rev_id = anna.revision.id
@@ -192,14 +187,16 @@
# Todo: Test for first revision on last page.
# Todo: Test for last revision minus 50 on second page.
# Page 1. (Implied id=1)
- offset = url_for(controller='revision', action='list')
- res = self.app.get(offset + '?format=atom')
+ offset = url_for(controller='revision', action='list', format='atom')
+ res = self.app.get(offset)
assert '<feed' in res, res
assert 'xmlns="http://www.w3.org/2005/Atom"' in res, res
assert '</feed>' in res, res
# Todo: Better test for 'days' request param.
# - fake some older revisions and check they aren't included.
- res = self.app.get(offset + '?format=atom&days=30')
+ offset = url_for(controller='revision', action='list', format='atom',
+ days=30)
+ res = self.app.get(offset)
assert '<feed' in res, res
assert 'xmlns="http://www.w3.org/2005/Atom"' in res, res
assert '</feed>' in res, res
--- a/ckan/tests/pylons_controller.py Fri May 06 18:00:57 2011 +0100
+++ b/ckan/tests/pylons_controller.py Fri May 06 18:03:11 2011 +0100
@@ -14,6 +14,9 @@
from ckan.tests import *
class MockTranslator(object):
+ def gettext(self, value):
+ return value
+
def ugettext(self, value):
return value
http://bitbucket.org/okfn/ckan/changeset/0024f2db026b/
changeset: r3067:0024f2db026b
branch: feature-1078-refactor-wui-to-use-logic-layer
user: kindly
date: 2011-05-10 12:26:32
summary: [forms] group form finished
affected #: 19 files (10.5 KB)
--- a/ckan/config/routing.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/config/routing.py Tue May 10 11:26:32 2011 +0100
@@ -190,6 +190,10 @@
# group
map.redirect("/groups", "/group")
map.redirect("/groups/{url:.*}", "/group/{url}")
+
+ map.connect('/group/new', controller='group_logic', action='new')
+ map.connect('/group/edit/{id}', controller='group_logic', action='edit')
+
map.connect('/group', controller='group', action='index')
map.connect('/group/list', controller='group', action='list')
map.connect('/group/new', controller='group', action='new')
--- a/ckan/controllers/api.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/controllers/api.py Tue May 10 11:26:32 2011 +0100
@@ -154,7 +154,7 @@
action_map = {
'revision': get.revision_show,
- 'group': get.group_show,
+ 'group': get.group_show_rest,
'tag': get.tag_show,
'package': get.package_show_rest,
('package', 'relationships'): get.package_relationships_list,
@@ -174,6 +174,7 @@
response.status_int = 400
return gettext('Cannot read entity of this type: %s') % register
try:
+
return self._finish_ok(action(context))
except NotFound, e:
extra_msg = e.extra_msg
@@ -188,13 +189,15 @@
action_map = {
('package', 'relationships'): create.package_relationship_create,
- 'group': create.group_create,
+ 'group': create.group_create_rest,
'package': create.package_create_rest,
'rating': create.rating_create,
}
+
for type in model.PackageRelationship.get_all_types():
action_map[('package', type)] = create.package_relationship_create
+
context = {'model': model, 'session': model.Session, 'user': c.user,
'id': id, 'id2': id2, 'rel': subregister,
'api_version': ver}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckan/controllers/group_logic.py Tue May 10 11:26:32 2011 +0100
@@ -0,0 +1,113 @@
+
+import logging
+
+from ckan.lib.base import BaseController, render, c, model, abort, request
+from ckan.lib.base import redirect, _, config, h
+import ckan.logic.action.create as create
+import ckan.logic.action.update as update
+import ckan.logic.action.get as get
+from ckan.lib.navl.dictization_functions import DataError, unflatten
+from ckan.logic import NotFound, NotAuthorized, ValidationError
+from ckan.logic.schema import group_form_schema
+from ckan.logic import tuplize_dict, clean_dict
+from ckan.authz import Authorizer
+
+log = logging.getLogger(__name__)
+
+class GroupLogicController(BaseController):
+
+ def new(self, data=None, errors=None, error_summary=None):
+ context = {'model': model, 'session': model.Session,
+ 'user': c.user or c.author, 'extras_as_string': True,
+ 'schema': group_form_schema(),
+ 'save': 'save' in request.params}
+
+ auth_for_create = Authorizer().am_authorized(c, model.Action.GROUP_CREATE, model.System())
+ if not auth_for_create:
+ abort(401, _('Unauthorized to create a group'))
+
+ if context['save'] and not data:
+ return self._save_new(context)
+
+ data = data or {}
+ errors = errors or {}
+ error_summary = error_summary or {}
+ vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
+
+ self._setup_template_variables(context)
+ c.form = render('group/new_group_form.html', extra_vars=vars)
+ return render('group/new.html')
+
+ def edit(self, id, data=None, errors=None, error_summary=None):
+ context = {'model': model, 'session': model.Session,
+ 'user': c.user or c.author, 'extras_as_string': True,
+ 'save': 'save' in request.params,
+ 'schema': group_form_schema(),
+ 'id': id}
+
+ if context['save'] and not data:
+ return self._save_edit(id, context)
+
+ try:
+ old_data = get.group_show(context)
+ c.grouptitle = old_data.get('title')
+ c.groupname = old_data.get('name')
+ data = data or old_data
+ except NotAuthorized:
+ abort(401, _('Unauthorized to read group %s') % '')
+
+ group = context.get("group")
+
+ am_authz = self.authorizer.am_authorized(c, model.Action.EDIT, group)
+ if not am_authz:
+ abort(401, _('User %r not authorized to edit %s') % (c.user, id))
+
+ errors = errors or {}
+ vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
+
+ self._setup_template_variables(context)
+ c.form = render('group/new_group_form.html', extra_vars=vars)
+ return render('group/edit.html')
+
+ def _save_new(self, context):
+ try:
+ data_dict = clean_dict(unflatten(tuplize_dict(dict(request.params))))
+ context['message'] = data_dict.get('log_message', '')
+ group = create.group_create(data_dict, context)
+ h.redirect_to(controller='group', action='read', id=group['name'])
+ except NotAuthorized:
+ abort(401, _('Unauthorized to read group %s') % '')
+ except NotFound, e:
+ abort(404, _('Package not found'))
+ except DataError:
+ abort(400, _(u'Integrity Error'))
+ except ValidationError, e:
+ errors = e.error_dict
+ error_summary = e.error_summary
+ return self.new(data_dict, errors, error_summary)
+
+ def _save_edit(self, id, context):
+ try:
+ data_dict = clean_dict(unflatten(tuplize_dict(dict(request.params))))
+ context['message'] = data_dict.get('log_message', '')
+ group = update.group_update(data_dict, context)
+ h.redirect_to(controller='group', action='read', id=group['name'])
+ except NotAuthorized:
+ abort(401, _('Unauthorized to read group %s') % id)
+ except NotFound, e:
+ abort(404, _('Package not found'))
+ except DataError:
+ abort(400, _(u'Integrity Error'))
+ except ValidationError, e:
+ errors = e.error_dict
+ error_summary = e.error_summary
+ return self.edit(id, data_dict, errors, error_summary)
+
+ def _setup_template_variables(self, context):
+ c.is_sysadmin = Authorizer().is_sysadmin(c.user)
+
+ ## This is messy as auths take domain object not data_dict
+ group = context.get('group') or c.pkg
+ if group:
+ c.auth_for_change_state = Authorizer().am_authorized(
+ c, model.Action.CHANGE_STATE, group)
--- a/ckan/controllers/package_logic.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/controllers/package_logic.py Tue May 10 11:26:32 2011 +0100
@@ -16,7 +16,6 @@
class PackageLogicController(BaseController):
-
def new(self, data=None, errors=None, error_summary=None):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
--- a/ckan/lib/dictization/model_dictize.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/lib/dictization/model_dictize.py Tue May 10 11:26:32 2011 +0100
@@ -71,8 +71,8 @@
result_dict = table_dictize(group, context)
- result_dict["extras"] = obj_dict_dictize(
- group._extras, context, lambda x: x["key"])
+ result_dict["extras"] = extras_dict_dictize(
+ group._extras, context)
result_dict["packages"] = obj_list_dictize(
group.packages, context)
--- a/ckan/lib/dictization/model_save.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/lib/dictization/model_save.py Tue May 10 11:26:32 2011 +0100
@@ -45,7 +45,7 @@
return obj_list
-def package_extras_save(extras_dicts, pkg, context):
+def extras_save(extras_dicts, context):
model = context["model"]
session = context["session"]
@@ -53,7 +53,7 @@
result_dict = {}
for extra_dict in extras_dicts:
- if extra_dict.get("delete"):
+ if extra_dict.get("deleted"):
continue
if extras_as_string:
result_dict[extra_dict["key"]] = extra_dict["value"]
@@ -62,17 +62,6 @@
return result_dict
-def group_extras_save(extras_dicts, pkg, context):
-
- model = context["model"]
- session = context["session"]
-
- obj_dict = {}
- for extra_dict in extras_dicts:
- obj = table_dict_save(extra_dict, model.GroupExtra, context)
- obj_dict[extra_dict["key"]] = obj
-
- return obj_dict
def tag_list_save(tag_dicts, context):
@@ -84,7 +73,7 @@
obj = table_dict_save(table_dict, model.Tag, context)
tag_list.append(obj)
- return tag_list
+ return list(set(tag_list))
def group_list_save(group_dicts, context):
@@ -147,7 +136,7 @@
if objects or not allow_partial_update:
pkg.relationships_as_object[:] = relationship_list_save(objects, context)
- extras = package_extras_save(pkg_dict.get("extras", {}), pkg, context)
+ extras = extras_save(pkg_dict.get("extras", {}), context)
if extras or not allow_partial_update:
old_extras = set(pkg.extras.keys())
new_extras = set(extras.keys())
@@ -164,18 +153,22 @@
model = context["model"]
session = context["session"]
group = context.get("group")
+ allow_partial_update = context.get("allow_partial_update", False)
+
Group = model.Group
Package = model.Package
if group:
group_dict["id"] = group.id
group = table_dict_save(group_dict, Group, context)
-
- extras = group_extras_save(group_dict.get("extras", []), group, context)
-
- group._extras.clear()
- for key, value in extras.iteritems():
- group._extras[key] = value
+ extras = extras_save(group_dict.get("extras", {}), context)
+ if extras or not allow_partial_update:
+ old_extras = set(group.extras.keys())
+ new_extras = set(extras.keys())
+ for key in old_extras - new_extras:
+ del group.extras[key]
+ for key in new_extras:
+ group.extras[key] = extras[key]
package_dicts = group_dict.get("packages", [])
@@ -187,10 +180,12 @@
if id:
pkg = session.query(Package).get(id)
if not pkg:
- pkg = session.query(Package).filter_by(name=package["name"]).one()
- packages.append(pkg)
+ pkg = session.query(Package).filter_by(name=package["name"]).first()
+ if pkg:
+ packages.append(pkg)
- group.packages[:] = packages
+ if packages or not allow_partial_update:
+ group.packages[:] = packages
return group
--- a/ckan/logic/action/create.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/logic/action/create.py Tue May 10 11:26:32 2011 +0100
@@ -6,7 +6,11 @@
IPackageController)
from ckan.logic import NotFound, check_access, NotAuthorized, ValidationError
from ckan.lib.base import _
-from ckan.lib.dictization.model_dictize import package_to_api1, package_to_api2
+from ckan.lib.dictization.model_dictize import (package_to_api1,
+ package_to_api2,
+ group_to_api1,
+ group_to_api2)
+
from ckan.lib.dictization.model_save import (group_api_to_dict,
group_dict_save,
package_api_to_dict,
@@ -22,6 +26,7 @@
from ckan.lib.navl.dictization_functions import validate
from ckan.logic.action.update import (_update_package_relationship,
package_error_summary,
+ group_error_summary,
check_group_auth)
log = logging.getLogger(__name__)
@@ -30,7 +35,6 @@
model = context['model']
user = context['user']
preview = context.get('preview', False)
- flat = context.get('flat', False)
schema = context.get('schema') or default_create_package_schema()
model.Session.remove()
@@ -120,22 +124,23 @@
def group_create(data_dict, context):
model = context['model']
user = context['user']
+ schema = context.get('schema') or default_group_schema()
check_access(model.System(), model.Action.GROUP_CREATE, context)
- context = {'model': model, 'session': model.Session}
- dictized = group_api_to_dict(data_dict, context)
-
- data, errors = validate(dictized,
- default_group_schema(),
- context)
+ data, errors = validate(data_dict, schema, context)
if errors:
- raise ValidationError(errors)
+ model.Session.rollback()
+ raise ValidationError(errors, group_error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
- rev.message = _(u'REST API: Create object %s') % data['name']
+
+ if 'message' in context:
+ rev.message = context['message']
+ else:
+ rev.message = _(u'REST API: Create object %s') % data.get("name")
group = group_dict_save(data, context)
@@ -147,6 +152,7 @@
for item in PluginImplementations(IGroupController):
item.create(group)
model.repo.commit()
+ context["group"] = group
context["id"] = group.id
log.debug('Created object %s' % str(group.name))
return group_dictize(group, context)
@@ -195,7 +201,7 @@
dictized_package = package_api_to_dict(data_dict, context)
dictized_after = package_create(dictized_package, context)
- pkg = context["package"]
+ pkg = context['package']
if api == '1':
package_dict = package_to_api1(pkg, context)
@@ -204,3 +210,19 @@
return package_dict
+def group_create_rest(data_dict, context):
+
+ api = context.get('api_version') or '1'
+
+ dictized_group = group_api_to_dict(data_dict, context)
+ dictized_after = group_create(dictized_group, context)
+
+ group = context['group']
+
+ if api == '1':
+ group_dict = group_to_api1(group, context)
+ else:
+ group_dict = group_to_api2(group, context)
+
+ return group_dict
+
--- a/ckan/logic/action/get.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/logic/action/get.py Tue May 10 11:26:32 2011 +0100
@@ -5,7 +5,10 @@
import ckan.authz
from ckan.lib.dictization.model_dictize import group_to_api1, group_to_api2
-from ckan.lib.dictization.model_dictize import package_to_api1, package_to_api2, package_dictize
+from ckan.lib.dictization.model_dictize import (package_to_api1,
+ package_to_api2,
+ package_dictize,
+ group_dictize)
def package_list(context):
@@ -159,20 +162,20 @@
id = context['id']
api = context.get('api_version') or '1'
+
group = model.Group.get(id)
+ context['group'] = group
+
if group is None:
raise NotFound
+ check_access(group, model.Action.READ, context)
- check_access(group, model.Action.READ, context)
+ group_dict = group_dictize(group, context)
for item in PluginImplementations(IGroupController):
item.read(group)
- if api == '2':
- _dict = group_to_api2(group, context)
- else:
- _dict = group_to_api1(group, context)
- #TODO check it's not none
- return _dict
+
+ return group_dict
def tag_show(context):
@@ -202,3 +205,17 @@
package_dict = package_to_api2(pkg, context)
return package_dict
+
+def group_show_rest(context):
+
+ group_show(context)
+ api = context.get('api_version') or '1'
+ group = context['group']
+
+ if api == '2':
+ group_dict = group_to_api2(group, context)
+ else:
+ group_dict = group_to_api1(group, context)
+
+ return group_dict
+
--- a/ckan/logic/action/update.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/logic/action/update.py Tue May 10 11:26:32 2011 +0100
@@ -33,6 +33,18 @@
error_summary[_(prettify(key))] = error[0]
return error_summary
+def group_error_summary(error_dict):
+
+ error_summary = {}
+ for key, error in error_dict.iteritems():
+ if key == 'extras':
+ error_summary[_('Extras')] = _('Missing Value')
+ elif key == 'extras_validation':
+ error_summary[_('Extras')] = error[0]
+ else:
+ error_summary[_(prettify(key))] = error[0]
+ return error_summary
+
def check_group_auth(data_dict, context):
model = context['model']
pkg = context.get("package")
@@ -145,15 +157,12 @@
return _update_package_relationship(entity, comment, context)
-def group_update_rest(data_dict, context):
-
- dictized = group_api_to_dict(data_dict, context)
- return group_update(dictized, context)
def group_update(data_dict, context):
model = context['model']
user = context['user']
+ schema = context.get('schema') or default_update_group_schema()
id = context['id']
group = model.Group.get(id)
@@ -163,17 +172,20 @@
check_access(group, model.Action.EDIT, context)
- data, errors = validate(data_dict,
- default_update_group_schema(),
- context)
+ data, errors = validate(data_dict, schema, context)
if errors:
- raise ValidationError(errors)
+ model.Session.rollback()
+ raise ValidationError(errors, group_error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
+ if 'message' in context:
+ rev.message = context['message']
+ else:
+ rev.message = _(u'REST API: Create object %s') % data.get("name")
+
group = group_dict_save(data, context)
- rev.message = _(u'REST API: Update object %s') % group.name
for item in PluginImplementations(IGroupController):
item.edit(group)
@@ -196,3 +208,13 @@
dictized_package = package_api_to_dict(data_dict, context)
return package_update(dictized_package, context)
+def group_update_rest(data_dict, context):
+
+ model = context['model']
+ id = context["id"]
+ group = model.Group.get(id)
+ context["group"] = group
+ context["allow_partial_update"] = True
+ dictized_package = group_api_to_dict(data_dict, context)
+ return group_update(dictized_package, context)
+
--- a/ckan/logic/schema.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/logic/schema.py Tue May 10 11:26:32 2011 +0100
@@ -57,7 +57,7 @@
schema = {
'id': [ignore_missing, unicode, package_id_exists],
- 'revision_id': [ignore_missing, unicode],
+ 'revision_id': [ignore],
'name': [not_empty, unicode, name_validator, package_name_validator],
'title': [if_empty_same_as("name"), unicode],
'author': [ignore_missing, unicode],
@@ -80,7 +80,6 @@
'groups': {
'id': [ignore_missing, unicode],
'__extras': [ignore],
- 'keep': [ignore_missing, unicode],
}
}
return schema
@@ -109,7 +108,6 @@
schema['groups'] = {
'id': [not_empty, unicode],
'__extras': [empty],
- 'keep': [ignore_missing, unicode],
}
schema['tag_string'] = [ignore_missing, tag_string_convert]
schema['extras_validation'] = [duplicate_extras_key, ignore]
@@ -129,13 +127,14 @@
schema = {
'id': [ignore_missing, unicode],
- 'revision_id': [ignore_missing, unicode],
+ 'revision_id': [ignore],
'name': [not_empty, unicode, name_validator, group_name_validator],
'title': [ignore_missing, unicode],
'description': [ignore_missing, unicode],
'state': [ignore],
'created': [ignore],
'extras': default_extras_schema(),
+ '__extras': [ignore],
'packages': {
"id": [not_empty, unicode, package_id_or_name_exists],
"__extras": [ignore]
@@ -143,11 +142,22 @@
}
return schema
+def group_form_schema():
+ schema = default_group_schema()
+ #schema['extras_validation'] = [duplicate_extras_key, ignore]
+ schema['packages'] = {
+ "name": [not_empty, unicode],
+ "__extras": [ignore]
+ }
+ return schema
+
+
def default_update_group_schema():
schema = default_group_schema()
schema["name"] = [ignore_missing, group_name_validator, unicode]
return schema
+
def default_extras_schema():
schema = {
@@ -155,7 +165,7 @@
'key': [not_empty, unicode],
'value': [not_missing, unicode],
'state': [ignore],
- 'delete': [ignore_missing],
+ 'deleted': [ignore_missing],
}
return schema
--- a/ckan/logic/validators.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/logic/validators.py Tue May 10 11:26:32 2011 +0100
@@ -93,7 +93,7 @@
extras = unflattened.get('extras', [])
extras_keys = []
for extra in extras:
- if not extra.get('delete'):
+ if not extra.get('deleted'):
extras_keys.append(extra['key'])
for extra_key in set(extras_keys):
--- a/ckan/public/scripts/autocompleter.js Fri May 06 18:03:11 2011 +0100
+++ b/ckan/public/scripts/autocompleter.js Tue May 10 11:26:32 2011 +0100
@@ -2,33 +2,27 @@
(function () {
- function extractDataAttributes() {
- var el = $(this);
- $.each(this.attributes, function () {
- var m = this.name.match(/data\-(\S+)/);
- if (m) { el.data(m[1], this.value); }
- });
- }
-
function processResult(e, item) {
- console.log(item);
- $(this).val('')
- .parent('dd').before(
- '<input type="hidden" name="PackageGroup--package_name" value="' + item[1] + '">' +
+ var input_box = $(this)
+ input_box.val('')
+ var parent_dd = input_box.parent('dd')
+ var old_name = input_box.attr('name')
+ var field_name_regex = /^(\S+)__(\d+)__(\S+)$/;
+ var split = old_name.match(field_name_regex)
+
+ var new_name = split[1] + '__' + (parseInt(split[2]) + 1) + '__' + split[3]
+
+ input_box.attr('name', new_name)
+ input_box.attr('id', new_name)
+
+ parent_dd.before(
+ '<input type="hidden" name="' + old_name + '" value="' + item[1] + '">' +
'<dd>' + item[0] + '</dd>'
);
}
$(document).ready(function () {
- $('input.autocomplete').each(function () {
- extractDataAttributes.apply(this);
-
- var url = $(this).data('autocomplete-url');
-
- if (url) {
- $(this).autocomplete(url, {})
+ $('input.autocomplete').autocomplete('/package/autocomplete', {})
.result(processResult);
- }
- });
});
})(jQuery);
--- a/ckan/public/scripts/flexitable.js Fri May 06 18:03:11 2011 +0100
+++ b/ckan/public/scripts/flexitable.js Tue May 10 11:26:32 2011 +0100
@@ -6,7 +6,7 @@
(function ($) {
- var fieldNameRegex = /^(\S+)-(\d+)-(\S+)$/;
+ var fieldNameRegex = /^(\S+)__(\d+)__(\S+)$/;
var controlsHtml = '<td><div class="controls">' +
'<a class="moveUp" title="Move this row up" href="#moveUp">Move up</a>' +
@@ -24,8 +24,8 @@
function setRowNumber(tr, num) {
$(tr).find('input').each(function () {
$(this).attr({
- id: $(this).attr('id').replace(fieldNameRegex, "$1-" + num + "-$3"),
- name: $(this).attr('name').replace(fieldNameRegex, "$1-" + num + "-$3")
+ id: $(this).attr('id').replace(fieldNameRegex, "$1__" + num + "__$3"),
+ name: $(this).attr('name').replace(fieldNameRegex, "$1__" + num + "__$3")
});
});
}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckan/templates/group/new_group_form.html Tue May 10 11:26:32 2011 +0100
@@ -0,0 +1,90 @@
+<form id="group-edit" action="" method="post"
+ py:attrs="{'class':'has-errors'} if errors else {}"
+ xmlns:i18n="http://genshi.edgewall.org/i18n"
+ xmlns:py="http://genshi.edgewall.org/"
+ xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<div class="error-explanation" py:if="error_summary">
+<h2>Errors in form</h2>
+<p>The form contains invalid entries:</p>
+<ul>
+ <li py:for="key, error in error_summary.items()">${"%s: %s" % (key, error)}</li>
+</ul>
+</div>
+
+<fieldset>
+ <legend>Details</legend>
+ <dl>
+ <dt><label class="field_opt" for="name">Name *</label></dt>
+ <dd><input id="name" name="name" type="text" value="${data.get('name', '')}"/></dd>
+ <dd class="instructions basic"><br/><strong>Unique identifier</strong> for group.<br/>2+ chars, lowercase, using only 'a-z0-9' and '-_'</dd>
+ <dd class="field_error" py:if="errors.get('name', '')">${errors.get('name', '')}</dd>
+
+ <dt><label class="field_opt" for="title">Title</label></dt>
+ <dd><input id="title" name="title" type="text" value="${data.get('title', '')}"/></dd>
+ <dd class="field_error" py:if="errors.get('title', '')">${errors.get('title', '')}</dd>
+
+ <dt><label class="field_opt" for="title">Description</label></dt>
+ <dd><textarea cols="60" id="description" name="description" rows="15">${data.get('description', '')}</textarea></dd>
+
+ <dt py:if="c.is_sysadmin or c.auth_for_change_state"><label class="field_opt" for="state">State</label></dt>
+ <dd py:if="c.is_sysadmin or c.auth_for_change_state">
+ <select id="state" name="state" >
+ <option py:attrs="{'selected': 'selected' if data.get('state') == 'active' else None}" value="active">active</option>
+ <option py:attrs="{'selected': 'selected' if data.get('state') == 'deleted' else None}" value="deleted">deleted</option>
+ </select>
+ </dd>
+ </dl>
+</fieldset>
+
+<fieldset>
+ <legend>Extras</legend>
+ <dl>
+ <py:with vars="extras = data.get('extras', [])">
+ <py:for each="num, extra in enumerate(data.get('extras', []))">
+ <dt><label for="extras__${num}__value">${extra.get('key')}</label></dt>
+ <dd>
+ <input id="extras__${num}__key" name="extras__${num}__key" type="hidden" value="${extra.get('key')}" />
+ <input id="extras__${num}__value" name="extras__${num}__value" type="text" value="${extra.get('value')}" />
+ <input type="checkbox" name="extras__${num}__deleted" checked="${extra.get('deleted')}">Delete</input>
+ </dd>
+ </py:for>
+
+ <py:for each="num in range(len(extras), len(extras) + 4)">
+ <dt><label for="extras__${num}__key">New key</label></dt>
+ <dd>
+ <input class="medium-width" id="extras__${num}__key" name="extras__${num}__key" type="text" />
+ with value
+ <input class="medium-width" id="extras__${num}__value" name="extras__${num}__value" type="text" />
+ </dd>
+ </py:for>
+ </py:with>
+ </dl>
+</fieldset>
+
+<fieldset>
+ <legend>Packages</legend>
+ <dl py:if="data.get('packages')">
+ <py:for each="num, package in enumerate(data.get('packages'))">
+ <dt><input checked="checked" id="packages__${num}__name" name="packages__${num}__name" type="checkbox" value="${package['name']}"/></dt>
+ <dd>
+ <label for="packages__${num}__name">${package['name']}</label>
+ </dd>
+ </py:for>
+ </dl>
+ <p py:if="not data.get('packages')">There are no packages currently in this group.</p>
+</fieldset>
+
+<fieldset>
+ <legend>
+ Add packages
+ </legend>
+ <dl>
+ <dt><label class="field_opt" for="packages__${len(data.get('packages', []))}__name">Package</label></dt>
+ <dd><input class="autocomplete" id="packages__${len(data.get('packages', []))}__name" name="packages__${len(data.get('packages', []))}__name" type="text" /></dd>
+ </dl>
+</fieldset>
+
+ <br />
+ <input id="save" name="save" type="submit" value="Save" />
+</form>
--- a/ckan/templates/package/new.html Fri May 06 18:03:11 2011 +0100
+++ b/ckan/templates/package/new.html Tue May 10 11:26:32 2011 +0100
@@ -21,6 +21,8 @@
<!-- Tagcomplete --><script type="text/javascript" src="${g.site_url}/scripts/tagcomplete.js"></script><link rel="stylesheet" href="${g.site_url}/css/tagcomplete.css" />
+
+ <xi:include href="new_package_form.js"/></py:def><div py:match="content">
--- a/ckan/templates/package/new_package_form.html Fri May 06 18:03:11 2011 +0100
+++ b/ckan/templates/package/new_package_form.html Tue May 10 11:26:32 2011 +0100
@@ -4,7 +4,6 @@
xmlns:py="http://genshi.edgewall.org/"
xmlns:xi="http://www.w3.org/2001/XInclude">
- <xi:include href="new_package_form.js"/><div class="error-explanation" py:if="error_summary"><h2>Errors in form</h2>
@@ -15,10 +14,7 @@
</div><fieldset>
- <legend>
- Basic information
- </legend>
-
+ <legend> Basic information</legend><dl><dt><label class="field_opt" for="title">Title</label></dt><dd><input id="title" name="title" type="text" value="${data.get('title', '')}"/></dd>
@@ -58,7 +54,7 @@
<dt><label class="field_opt" for="tags">Tags</label></dt><dd><input class="long tagComplete" data-tagcomplete-queryparam="incomplete"
- data-tagcomplete-url="/apiv2/package/autocomplete" id="tag_string" name="tag_string" size="60" type="text"
+ data-tagcomplete-url="/api/2/util/tag/autocomplete" id="tag_string" name="tag_string" size="60" type="text"
value="${data.get('tag_string') or ' '.join([tag['name'] for tag in data.get('tags', [])])}" /></dd><dd class="instructions basic">Terms that may link this dataset to similar ones. For more information on conventions, see <a href="http://wiki.okfn.org/ckan/doc/faq#TagConventions">this wiki page</a>.</dd>
@@ -155,16 +151,14 @@
<fieldset><legend>Extras</legend>
-
<dl>
-
<py:with vars="extras = data.get('extras', [])"><py:for each="num, extra in enumerate(data.get('extras', []))"><dt><label for="extras__${num}__value">${extra.get('key')}</label></dt><dd><input id="extras__${num}__key" name="extras__${num}__key" type="hidden" value="${extra.get('key')}" /><input id="extras__${num}__value" name="extras__${num}__value" type="text" value="${extra.get('value')}" />
- <input type="checkbox" name="extras__${num}__delete" checked="${extra.get('delete')}">Delete</input>
+ <input type="checkbox" name="extras__${num}__deleted" checked="${extra.get('deleted')}">Delete</input></dd></py:for>
--- a/ckan/tests/functional/test_group.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/tests/functional/test_group.py Tue May 10 11:26:32 2011 +0100
@@ -147,8 +147,8 @@
form = res.forms['group-edit']
group = model.Group.by_name(self.groupname)
- titlefn = 'Group-%s-title' % group.id
- descfn = 'Group-%s-description' % group.id
+ titlefn = 'title'
+ descfn = 'description'
newtitle = 'xxxxxxx'
newdesc = '''### Lots of stuff here
@@ -158,7 +158,7 @@
form[titlefn] = newtitle
form[descfn] = newdesc
pkg = model.Package.by_name(self.packagename)
- form['PackageGroup--package_name'] = pkg.name
+ form['packages__2__name'] = pkg.name
res = form.submit('save', status=302, extra_environ={'REMOTE_USER': 'russianfan'})
@@ -201,7 +201,7 @@
res = self.app.get(offset, status=200, extra_environ={'REMOTE_USER': 'russianfan'})
form = res.forms['group-edit']
group = model.Group.by_name(self.groupname)
- form['Group-%s-title' % group.id] = "huhuhu"
+ form['title'] = "huhuhu"
res = form.submit('save', status=302, extra_environ={'REMOTE_USER': 'russianfan'})
assert plugin.calls['edit'] == 1, plugin.calls
plugins.unload(plugin)
@@ -233,7 +233,7 @@
assert res.request.url.startswith('/user/login')
def test_2_new(self):
- prefix = 'Group--'
+ prefix = ''
group_name = u'testgroup'
group_title = u'Test Title'
group_description = u'A Description'
@@ -246,14 +246,14 @@
assert fv[prefix+'name'].value == '', fv.fields
assert fv[prefix+'title'].value == ''
assert fv[prefix+'description'].value == ''
- assert fv['PackageGroup--package_name'].value == '', fv['PackageGroup--package_name'].value
+ assert fv['packages__0__name'].value == '', fv['PackageGroup--package_name'].value
# Edit form
fv[prefix+'name'] = group_name
fv[prefix+'title'] = group_title
fv[prefix+'description'] = group_description
pkg = model.Package.by_name(self.packagename)
- fv['PackageGroup--package_name'] = pkg.name
+ fv['packages__0__name'] = pkg.name
res = fv.submit('save', status=302, extra_environ={'REMOTE_USER': 'russianfan'})
res = res.follow()
assert '%s' % group_title in res, res
@@ -267,7 +267,7 @@
assert group.packages == [pkg]
def test_3_new_duplicate(self):
- prefix = 'Group--'
+ prefix = ''
# Create group
group_name = u'testgrp1'
@@ -302,8 +302,8 @@
offset = url_for(controller='group', action='new')
res = self.app.get(offset, status=200, extra_environ={'REMOTE_USER': 'russianfan'})
form = res.forms['group-edit']
- form['Group--name'] = "hahaha"
- form['Group--title'] = "huhuhu"
+ form['name'] = "hahaha"
+ form['title'] = "huhuhu"
res = form.submit('save', status=302, extra_environ={'REMOTE_USER': 'russianfan'})
assert plugin.calls['create'] == 1, plugin.calls
plugins.unload(plugin)
--- a/ckan/tests/functional/test_package.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/tests/functional/test_package.py Tue May 10 11:26:32 2011 +0100
@@ -202,7 +202,7 @@
self.check_tag(main_res, 'extras__%s__key' % num, key_in_html_body)
self.check_tag(main_res, 'extras__%s__value' % num, value_escaped)
if deleted:
- self.check_tag(main_res, 'extras__%s__delete' % num, 'checked')
+ self.check_tag(main_res, 'extras__%s__deleted' % num, 'checked')
assert params['log_message'] in main_res, main_res
@@ -706,7 +706,7 @@
fv[prefix+'extras__0__value'] = extra_changed[1].encode('utf8')
fv[prefix+'extras__3__key'] = extra_new[0].encode('utf8')
fv[prefix+'extras__3__value'] = extra_new[1].encode('utf8')
- fv[prefix+'extras__2__delete'] = True
+ fv[prefix+'extras__2__deleted'] = True
fv['log_message'] = log_message
res = fv.submit('preview', extra_environ={'REMOTE_USER':'testadmin'})
assert not 'Error' in res, res
--- a/ckan/tests/lib/test_dictization_schema.py Fri May 06 18:03:11 2011 +0100
+++ b/ckan/tests/lib/test_dictization_schema.py Tue May 10 11:26:32 2011 +0100
@@ -141,7 +141,6 @@
'packages': sorted([{'id': group.packages[0].id},
{'id': group.packages[1].id,
}], key=lambda x:x["id"]),
- 'revision_id': group.revision_id,
'title': u"Dave's books"}
http://bitbucket.org/okfn/ckan/changeset/82c8846cf0d1/
changeset: r3068:82c8846cf0d1
branch: feature-1078-refactor-wui-to-use-logic-layer
user: kindly
date: 2011-05-10 12:27:25
summary: [merge] default
affected #: 9 files (37.7 KB)
--- a/MANIFEST.in Tue May 10 11:26:32 2011 +0100
+++ b/MANIFEST.in Tue May 10 11:27:25 2011 +0100
@@ -1,5 +1,6 @@
include ckan/config/deployment.ini_tmpl
recursive-include ckan/public *
+recursive-include ckan/config *.ini
recursive-include ckan/templates *
recursive-include ckan/requires *
recursive-include ckan *.ini
--- a/ckan/__init__.py Tue May 10 11:26:32 2011 +0100
+++ b/ckan/__init__.py Tue May 10 11:27:25 2011 +0100
@@ -1,4 +1,4 @@
-__version__ = '1.4a'
+__version__ = '1.4.1a'
__description__ = 'Comprehensive Knowledge Archive Network (CKAN) Software'
__long_description__ = \
'''The CKAN software is used to run the Comprehensive Knowledge Archive
@@ -15,6 +15,7 @@
'''
__license__ = 'AGPL'
+# The packaging system replies on this import, please do not remove it
try:
# Ths automatically modifies sys.path so that the CKAN versions of
# key dependencies are used instead of the ones already installed.
--- a/ckan/config/deployment.ini_tmpl Tue May 10 11:26:32 2011 +0100
+++ b/ckan/config/deployment.ini_tmpl Tue May 10 11:27:25 2011 +0100
@@ -22,6 +22,9 @@
beaker.session.secret = ${app_instance_secret}
app_instance_uuid = ${app_instance_uuid}
+# Add any CKAN plugins here. Note: This line is required to be here for
+# packaging, even if it is empty.
+ckan.plugins =
# If you'd like to fine-tune the individual locations of the cache data dirs
# for the Cache data, or the Session saves, un-comment the desired settings
@@ -167,17 +170,17 @@
# Logging configuration
[loggers]
-keys = root
+keys = root, ckan, ckanext
[handlers]
-keys = console
+keys = console, file
[formatters]
keys = generic
[logger_root]
-level = INFO
-handlers = console
+level = WARNING
+handlers = console, file
[logger_ckan]
level = DEBUG
@@ -201,7 +204,7 @@
class = handlers.RotatingFileHandler
formatter = generic
level = NOTSET
-args = ('/var/log/ckan/ckan.log', 'a', 2000000, 9)
+args = ('%(here)s/ckan.log', 'a', 2000000, 9)
[formatter_generic]
format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s
--- a/ckan/lib/search/common.py Tue May 10 11:26:32 2011 +0100
+++ b/ckan/lib/search/common.py Tue May 10 11:27:25 2011 +0100
@@ -148,7 +148,10 @@
if key in self.BOOLEAN_OPTIONS:
value = asbool(value)
elif key in self.INTEGER_OPTIONS:
- value = int(value)
+ try:
+ value = int(value)
+ except ValueError:
+ raise SearchError('Value for search option %r must be an integer but received %r' % (key, value))
self[key] = value
def __getattr__(self, name):
--- a/ckan/tests/functional/api/test_package_search.py Tue May 10 11:26:32 2011 +0100
+++ b/ckan/tests/functional/api/test_package_search.py Tue May 10 11:27:25 2011 +0100
@@ -246,6 +246,12 @@
assert len(res_dict['results']) == 1, res_dict
assert res_dict['results'][0]['name'] == 'warandpeace', res_dict['results'][0]['name']
+ def test_11_pagination_syntax_error(self):
+ offset = self.base_url + '?all_fields=1&tags=russian&offset=should_be_integer&limit=1&order_by=name' # invalid offset value
+ res = self.app.get(offset, status=400)
+ assert('integer' in res.body)
+ assert('offset' in res.body)
+
def test_12_all_packages_qjson(self):
query = {'q': ''}
json_query = self.dumps(query)
--- a/ckan/wsgi.py Tue May 10 11:26:32 2011 +0100
+++ b/ckan/wsgi.py Tue May 10 11:27:25 2011 +0100
@@ -1,4 +1,8 @@
#!/usr/bin/env python
+
+# The packaging system requires this script at ckan/wsgi.py, please do not
+# move it.
+
usage = """
WSGI Script for CKAN
====================
@@ -18,8 +22,8 @@
WSGIScriptAlias / /etc/ckan/dgu.py
dgu.py will load the Pylons config: dgu.ini (looking in the same directory.)
+"""
-"""
import os
import sys
from apachemiddleware import MaintenanceResponse
--- a/doc/configuration.rst Tue May 10 11:26:32 2011 +0100
+++ b/doc/configuration.rst Tue May 10 11:27:25 2011 +0100
@@ -350,3 +350,13 @@
With this example setting, visitors (any user who is not logged in) and logged in users can only read packages that get created (only sysadmins can edit).
Defaults: see in ckan/model/authz.py for: ``default_default_user_roles``
+
+
+plugins
+-------
+
+Example::
+
+ ckan.plugins = disqus synchronous_search datapreview googleanalytics stats storage admin follower
+
+Specify which CKAN extensions are to be enabled. If you specify an extension but have not installed the code then CKAN will not start. Format in a space separated list of the extension names. The extension name is the key in the [ckan.plugins] section of the extension's setup.py.
--- a/doc/deb.rst Tue May 10 11:26:32 2011 +0100
+++ b/doc/deb.rst Tue May 10 11:27:25 2011 +0100
@@ -1,84 +1,168 @@
-Packaging CKAN as Debian Files
-++++++++++++++++++++++++++++++
+CKAN's Approach to Dependencies
++++++++++++++++++++++++++++++++
-.. note ::
+WARNING: This document is still under development, use only if you are a member
+of the CKAN team who wishes to be an early adopter and are interested in
+experimenting with Debian packaging.
- This guide is a work in progress, please report any problems to the
- ckan-dev mailing list.
+.. contents ::
- It will need a second draft once packaging has stabilised a bit more.
+Abstract
+========
-Dependencies
-============
+A typical CKAN install can have many dependencies, in the form of required
+system software such as PostgreSQL, Python libraries such as SQLAlchemy and
+other CKAN extensions such as ``ckanext-qa`` or ``ckanext-harvest``.
-In order to pacakge CKAN and dependencies as Debian files you'll need the
-following tools:
+As such we have to deal with lots of interdependencies which are often
+different depending on the combinations of features a particular CKAN install
+requires.
+
+There are three audiences we are primarily targetting that our dependency
+approach is designed to support:
+
+* Interested parties who just want to get a CKAN instance installed quickly
+ and easily to test it or to begin contributing
+* CKAN developers who want to have editable versions of all of the dependant
+ libraries so that they can improve them
+* System administrators who want to be able to deploy and upgrade CKAN
+ instances quickly, easily and reliably
+* Deployment and test managers who want to be confident that the live system
+ is running off exactly the libraries they have tested and configured in
+ exactly the same way
+
+In order to support these three groups, we allow installation of CKAN in two ways:
+
+* Development install via ``pip`` as described in the `README.rst <../README.html>`_ file
+* Production install on Ubuntu Lucid 10.04 LTS via ``apt-get install``
+
+The old instructions for a `production deployment <../deployment.html>`_ can
+also be followed but these will be deprecated over time as the ``.deb``
+packaging approach becomes better documented and understood.
+
+Virutally all CKAN instances currently run on Ubuntu Lucid 10.04 LTS so we have
+decided that this is the only officially supported production deployment
+platform. Of course CKAN will also run on any modern Mac or Linux distribution
+but if you go down this route you'll need to do a development install.
+
+In this document I'll explain in detail how and why we package CKAN the way we
+do.
+
+
+Overview of CKAN's Structure from a Packaging Point of View
+===========================================================
+
+There are conceptually two main parts to any CKAN installation:
+
+* Python libraries such as CKAN itself, SQLAlchemy, owslib and all the CKAN
+ extensions libraries that a particular CKAN or customized CKAN rely on
+* The configuration and deployment scripts that lead to a running CKAN
+ application (which may be a vanilla CKAN or a client-specific version (eg
+ CKAN, CKAN-DGU, CKAN-DATANL etc)
+* Third party servers that CKAN relies on (eg Apache, PostgreSQL etc)
+
+Luckily all third party servers that CKAN relies on are all ready packaged in
+Ubuntu Lucid for us so we don't need to worry about packaging them. We do need
+to worry about configuring them though. Let's look more at the other two.
+
+Python Libraries
+ The Python libraries CKAN relies on all have their own interdependencies.
+ If the libraries have already been packaged as ``.deb`` files we don't need to worry about
+ them because their dependencies will already be specified in the package.
+ Other libraries usually have their dependencies specified as the ``install_requires`` line in their
+ ``setup.py`` files. For the sorts of libraries that are already available
+
+
+Configuration and Deployment
+
+
+Understanding The Dependency Difficulty
+---------------------------------------
+
+In the past
+
+
+
+The Three Different Requires Files for CKAN
+===========================================
+
+In order to support both a development install and a package-based install it
+is important that we package the same versions of libraries that we develop
+against. There are three categories of dependant Python libraries:
+
+``present``
+ Those that already exist as packages in Ubuntu Lucid
+
+``misssing``
+ Those that don't exist as packages in Ubuntu Lucid
+
+``conflict``
+ Those that have a version which is different from the version in Ubuntu
+ Lucid
+
+For each of these categories we have a file in the ``ckan`` source tree's
+``requires`` directory which you can view `here
+<https://bitbucket.org/okfn/ckan/src/default/requires/>`_.
+
+
+Understanding the ``lucid_present.txt`` File
+--------------------------------------------
+
+The Python dependencies listed in the ``lucid_present.txt`` file are ``pip``
+installable links to the source tree holding the exact versions of the Python
+dependencies that Ubuntu uses. By running the command below you get development
+copies of the same software that Ubuntu has packaged:
::
- sudo apt-get install -y python wget dh-make devscripts build-essential fakeroot cdbs
+ pip install --ignore-installed -r lucid_present.txt
-And depending on the source repositories of the things you are packaging you'll
-probably need:
+We never need to package software in the ``lucid_present.txt`` file because it
+already exists so most of the time you would just install it directly rather
+than running the command above to get source versions. You can see the packages
+you would need to install by looking at the comment at the top of the file. At
+the time of writing it reads:
::
- sudo apt-get install -y mercurial git-core subversion
+ # The CKAN dependencies are already in Lucid and should be installed via
+ # apt-get if you are on that platform. If you are using a different platform
+ # you can install these dependencies via pip instead.
+ #
+ # sudo apt-get install python-psycopg2 python-lxml python-sphinx
+ # sudo apt-get install python-pylons python-formalchemy python-repoze.who
+ # sudo apt-get install python-repoze.who-plugins python-tempita python-zope.interface
+Packaging Dependencies Listed in ``lucid_missing.txt``
+------------------------------------------------------
-Preparation
-===========
+.. note ::
-In order to build packages you'll need a directory with all of the source code
-for the various packages you want to pacakge checked out in a directory. You'll
-also need a copy of BuildKit installed.
+ These are already packaged, so you don't need to pacakge them yourself, this
+ section just describes how you *could* do if you wanted to.
-The easiest way to achieve this is to set up a virtual environment with
-BuildKit installed and use ``pip`` to install the dependencies in it for you.
+Python dependencies listed in the ``lucid_missing.txt`` file are ``pip``
+installable links to the source tree holding the exact versions of the Python
+dependencies that CKAN requries. We have an automatic build process which can
+take these entries and automatically generate Ubuntu packages for them. The
+resulting packages are then published to our CKAN apt repository so that they
+can be automatically installed in production environments.
+
+To follow the automatic build process to build the missing packages you can do this:
+
::
- wget http://pylonsbook.com/virtualenv.py
- python virtualenv.py missing
- cd missing
- bin/easy_install pip
- bin/pip install BuildKit
+ sudo apt-get install -y python wget dh-make devscripts build-essential fakeroot cdbs mercurial git-core subversion python-virtualenv
+ virtualenv missing
+ bin/pip install --ignore-installed -r lucid_missing.txt
+ bin/pip install Buildkit
-The idea is that all the dependencies you want to package are in a
-``lucid_missing.txt`` file as editable requirements with exact revisions
-specified. For example, if we want to package ApacheMiddleware we would have this line:
-
-::
-
- -e hg+https://hg.3aims.com/public/ApacheMiddleware@c5ab5c3169a5#egg=ApacheMiddleware
-
-Once your requirements are in place, install them like this:
-
-::
-
- bin/pip install -r lucid_missing.txt
-
-.. tip ::
-
- You can't just do this because pip ignores the ``-e`` for code that is
- installed from a package and fails to put it in ``src``:
-
- ::
-
- # Won't work
- -e ApacheMiddleware
-
- You can put the source directory manually in your virtual environment's
- ``src`` directory though if you need to though.
-
-Automatic Packaging
-===================
-
-The BuildKit script will build and place Debian packages in your ``missing``
+BuildKit script will build and place Debian packages in your ``missing``
directory. Make sure there is nothing in there that shouldn't be overwritten by
this script.
-To package everything automatically, run it like this:
+Now run the BuildKit command like this:
::
@@ -89,35 +173,366 @@
quit when you are done. Names, version numbers and dependencies are
automatically generated.
-You should find all your packages nicely created now.
+.. caution ::
-Manual Packaging
-================
+ Most of the time you will never use the automatic process above for lazy
+ batch packaging. You'll more likely generate a single package with explicit
+ version numners using the ``buildkit.deb`` command or build your package
+ manually. Both approaches are described later.
-If you want more control over version numbers and dependencies or you just want
-to package one thing you can do so like this:
+Packaging Conflicting Python dependencies from ``lucid_conflicts.txt``
+----------------------------------------------------------------------
+
+.. note ::
+
+ These are already packaged, so you don't need to pacakge them yourself, this
+ section just describes how you *could* do if you wanted to.
+
+Python packages where CKAN depends on a version that is different from the one
+in the Ubuntu Lucid repositories are handled slightly differently. If we were
+to simply package them up and make them available the same way we do with
+missing packages there is a slim chance that any existing software which used
+the other version of the libray would stop working. To avoid the risk of
+interfering with other software on the sytem we take the following approach:
+
+* Create a ``python-ckan-deps`` package with copies of all the libraries we need
+* Change the ``python-ckan`` library to automatically try to import
+ ``ckan_deps`` if it can and then adjust the Python's ``sys.path`` just for
+ this instance to use the versions of the libraries in ``python-ckan-deps`` in
+ preference to any other versions installed.
+
+In this way we can use any arbitrary versions, without introducing conflicts.
+
+.. caution ::
+
+ The ``repoze.who`` sets of libraries are nigh-on impossible to package in
+ this way so we don't actually package ``repoze.who.openid`` at all, even
+ though we need a slightly more recent version. This is such an edge case
+ though that you should just install it manually into the system Python
+ and not worry too much for the time being.
+
+To actually build the ``python-ckan-deps`` package we follow the semi-manual
+Python packaging approach described next. (The example in the next section is
+actually for a CKAN Python extension called ``python-ckanext-qa`` but the same
+process applies).
+
+
+Semi-Manual Python Packaging
+============================
+
+The easiest way to package a Python library is with a tool called BuildKit I
+wrote specfically for the purpose. This section describes how to use it, but
+even if you don't want to use BuildKit and prefer to understand the
+complexities yourself by reading the `Understanding .deb files`_ section,
+please read this section too so you at least understand the naming conventions
+we are using.
+
+::
+
+ pip install buildkit
+
+For each Python package you wish to build a ``.deb`` file for you run the
+``buildkit.deb`` command. Here's an example:
::
+
+ python -m buildkit.deb /path/to/virtualenv ckanext-qa 1.3~01+lucid http://ckan.org python-owslib python-ckanext-csw
+
+Let's break this down.
- python -m buildkit.deb /path/to/virtualenv ckan-deps 1.3 http://ckan.org python-dep-1 python-dep-2 ... etc
+``python -m buildkit.deb``
+ This is just how you invoke the command from the command line
-Version Numbers
-===============
+``/path/to/virtualenv``
+ I think this can just be the path to the directory containing the
+ installed source code directoty you wish to package, it doesn't
+ have to be a virtualenv does it?
+``ckanext-qa``
+ The lowercase Python package name of the ``.deb`` file to be created.
+
+
+``1.3~01+lucid``
+ The version number of the package. There are three parts to this:
+
+ ``1.3``
+ This should always match exactly the version number specified in the
+ ``setup.py`` file for the library being packaged.
+
+ ``~01``
+ This is an incrementing number (starting at 01 each time the version
+ number above changes) which you change every time you re-package the
+ same version of the code to force apt to recognise your new package
+ as being more recent than the old one, even if the underlying code
+ hasn't changed.
+
+ ``+lucid``
+ This is a string representing the Debian/Ubuntu distribution that the
+ package targets. The apt repository doesn't assign any meaning to it,
+ it is just that in order to eventually support more than one flavour
+ of Debian or Ubuntu, the packages for each must have different
+ filenames *in addition* to being in a separate part of the apt repo
+ so we begin this convention now.
+
+``http://ckan.org``
+ The homepage for the package, usually ckan.org for ckan extensions.
+
+``python-owslib python-ckanext-csw ... etc``
+
+ Any extra arguments are treated as the Debain names of dependencies. These
+ always begin ``python-`` for Python libraries and would usually follow
+ ``ckanext-`` for all CKAN extensions.
+
+ .. tip ::
+
+ You can also specify any other Debian
+ packages here that are dependcies of the software you are packaging but as
+ you'll see later it is usually best to add such dependencies to the
+ *packaged application*. See "Packaging CKAN Extensions" for more information.
+
+When you run the command you will get your ``.deb`` file created.
+
To release an upgrade of a package it must have a higher version number. There
is a chance you may want to release a more recent version of a package despite
the fact the underlying version number hasn't changed. For this reason, we
always add a ``~`` character followed by a two digit number to the end of the
-actual version number as specified in ``setup.py`` for the package.
+actual version number as specified in ``setup.py`` for the package.
-For example, the version number for CKAN may be ``1.4.0a~01``, producing a
-package named ``python-ckan_1.4.0a~01_amd64.deb``.
+For example, if the version number for the ``ckanext-qa`` package in the
+example above is ``1.3~01``, a package named
+``python-ckanext-qa_1.3~01_amd64.deb`` would be produced by the command we've
+looked at.
-Writing a ``ckan`` command
-==========================
+.. note ::
+
+ All pacakges that CKAN itself depends on are already packaged according to
+ the settings in the three ``requires`` files that from part of the ``ckan``
+ source distribution so you shouldn't need to use the approach above to
+ package any of them, you should only need to do this for your own extensions
+ or libraries they rely on which aren't already CKAN dependencies. See
+ "The Three Different Requires Files" for more information on how packaging
+ of the core CKAN dependencies is managed.
-For packages that don't represent Python libraries it is actually easier to
-build the ``.deb`` manually rather than using Debian's tools.
+Understanding ``.deb`` files
+============================
+
+Broad Structure
+---------------
+
+Naming Conventions
+------------------
+
+The base naming conventions we use for packages are as follows:
+
+``ckan``
+ Unstalls CKAN, PostgreSQL, Apache etc. It adds the ``ckan-instance-create`` command which is then the only thing you need to create a new instance.
+
+``python-ckan``
+ The CKAN Python library packaged from code at http://bitbucket.org/okfn/ckan
+
+``python-ckanext-*``
+ Any CKAN extensions (can be application extensions or library extensions)
+
+``ckan-*``
+ Installs a client specific CKAN application
+
+
+
+The ``postinst`` and ``postrm`` files
+-------------------------------------
+
+The ``control`` file
+--------------------
+
+Extra scripts and permissions
+-----------------------------
+
+Packaging Python libraries
+--------------------------
+
+
+
+Packaging CKAN Extensions
+=========================
+
+There are two types of CKAN extension:
+
+* Client Applications (eg ``ckanext-dgu``, ``ckanext-datanl`` etc)
+* Helpful libraries (eg ``ckanext-qa``, ``ckanext-harvest``, ``ckanext-queue`` etc)
+
+All CKAN extensions (whether client applications or helpful libraries) are
+Python libraries and therefore need packaging. Their ``.deb`` filenames are the
+same as the Python package names but are always prefixed with ``python-`` so
+that ``ckanext-dgu`` becomes ``python-ckanext-dgu`` when packaged as a ``.deb``
+and ``ckanext-harvest`` becomes ``python-ckanext-harvest`` etc.
+
+CKAN extensions which are also client applications generally need to be
+deployed and therefore need require Apache and PostgreSQL to be installed and
+configured correctly too. In addition to the *python* package we therefore also
+create an *application* package for the extension which is named ``ckan-``
+followed by the last part of the extension name. So for ``ckanext-dgu`` two
+packages are created named ``python-ckanext-dgu`` and ``ckan-dgu``. This naming
+may sound slightly inconsistent but it allows a user who wishes to install a
+DGU CKAN instance to just type the command below:
+
+::
+
+ sudo apt-get install ckan-dgu
+
+Usually the ``ckan`` package will be a dependency of the your client
+application CKAN extension. When the ``ckan`` package is installed it installs
+``python-ckan`` as a dependency as well as a series of scripts in ``/usr/bin``
+such as:
+
+``ckan-create-instance``
+ create a new CKAN instance
+
+``ckan-maintenance-mode``
+ put a CKAN intance into or out of maintenence mode (prevent POSTs from
+ the web user interface)
+
+In the simple cases, these scripts can then be used in your client application
+CKAN extension's ``posinst`` script to set up the custom instance. In more
+complex cases you may write a ``postinst`` script from scratch. The
+``postinst`` script then forms part of the package and is run by the apt system
+as part of the package installation or upgrade process to configure your CKAN
+instance.
+
+
+
+
+
+
+
+
+
+
+
+
+Before we look at how to actually create an apt repository for your packages
+and how to publish your packages to it, let's understand what a user of your
+package will do to install it.
+
+Understaning How a User Installs from an apt repository
+=======================================================
+
+A user will follow the following process:
+
+First create the file ``/etc/apt/sources.list.d/okfn.list`` with this line, replacing ``lucid`` with the correct repo you want to use:
+
+::
+
+ echo "deb http://apt-alpha.ckan.org/lucid lucid universe" | sudo tee /etc/apt/sources.list.d/okfn.list
+
+Then add the package key to say you trust packages from this repository:
+
+::
+
+ sudo apt-get install wget
+ wget -qO- http://apt-alpha.ckan.org/packages.okfn.key | sudo apt-key add -
+ sudo apt-get update
+
+Now you can not install a CKAN extension application, just like any other Debian package:
+
+::
+
+ sudo apt-get install ckan-dgu
+
+At this point you should have a running instance. You may need to copy across
+an existing database if you need your instance pre-populated with data.
+
+
+Setting up a CKAN Apt Repository
+================================
+
+Now you've seen what a user expects to be able to do, let's set up the
+infrastructure to make to make it happen.
+
+
+From Scratch
+------------
+
+Our set up is based on `Joseph Ruscio's set up
+<http://joseph.ruscio.org/blog/2010/08/19/setting-up-an-apt-repository/>`_ and
+will allow us to support multiple operating systems if we want to as well as
+multiple architectures. At the moment we only support Ubuntu Lucid amd64.
+
+To help with repository management we use the ``reprepro`` tool. Despite the fact that the repositories could be set up for different OSs and versions (eg ``lenny``, ``lucid`` etc) we need to make sure that the package names are still unique. This means that we always add the distribution to the version number when we package.
+
+
+The most important detail that AFAIK isn’t covered in any of the tutorials had to do with package naming conventions. The naive assumption (at least on my part) is that you’ll have a different build of your package for each distro/arch combination, and import them into your repository as such. In other words reprepro should track the distro/arch of each import. In actuality, each build’s <PACKAGE>_<VERSION>_<ARCH> must be unique, even though you specify the distro during the includedeb operation.
+
+
+
+
+The Easy Way
+------------
+
+Log into the existing CKAN apt repository server and copy an existing directory
+that already contains the packages you need. For example, to create a
+repository for a new ``ckanext-dgu`` instance you might do:
+
+::
+
+ cd /var/packages/
+ cp -pr lucid dgu-new
+
+At this point you have a brand new repo, you can add new packages to it like this:
+
+::
+
+ cd dgu-new
+ sudo reprepro includedeb lucid ~/*.deb
+
+You can remove them like this from the same directory:
+
+::
+
+ sudo reprepro remove lucid python-ckan
+
+Any time a change is made you will need to enter the passphrase for the key.
+
+
+Automatic Packaging
+===================
+
+The BuildKit script will build and place Debian packages in your ``missing``
+directory. Make sure there is nothing in there that shouldn't be overwritten by
+this script.
+
+
+Adding a Package to a Repository
+================================
+
+
+Packaging CKAN Itself
+=====================
+
+
+
+
+Why We use ``pip`` rather than ``install_requires``
+===================================================
+
+
+Packaging CKAN and its dependencies for a production install
+============================================================
+
+Installing a Packaged CKAN-based Site
+=====================================
+
+Testing Your Packaging in a VM
+==============================
+
+The Release Process
+===================
+
+
+
+
+
+Creating the CKAN Command
+=========================
Create a directory named ``ckan``. Then within it create a ``DEBIAN`` directory with three files:
@@ -177,19 +592,24 @@
``postinst`` command to set up Apache and PostgreSQL for the instance
automatically.
+
Setting up the Repositories
===========================
-Convert a Python package installed into a virtualenv into a Debian package automatically
-
-Usage:
+Build individual dependencies like this:
::
- python -m buildkit.deb . ckanext-csw 0.3~08 http://ckan.org python-ckanext-dgu python-owslib
- python -m buildkit.deb . ckanext-dgu 0.2~06 http://ckan.org python-ckan
- python -m buildkit.deb . ckanext-qa 0.1~09 http://ckan.org python-ckan
- python -m buildkit.deb . ckan 1.3.2~10 http://ckan.org python-routes python-vdm python-pylons python-genshi python-sqlalchemy python-repoze.who python-repoze.who-plugins python-pyutilib.component.core python-migrate python-formalchemy python-sphinx python-markupsafe python-setuptools python-psycopg2 python-licenses python-ckan-deps
+ python -m buildkit.deb . ckanext-importlib 0.1~02 http://ckan.org python-ckan
+ python -m buildkit.deb . owslib 0.3.2beta~03 http://ckan.org python-lxml
+
+ python -m buildkit.deb . ckanext-inspire 0.1~03 http://ckan.org python-ckan
+ python -m buildkit.deb . ckanext-spatial 0.1~04 http://ckan.org python-ckan
+ python -m buildkit.deb . ckanext-harvest 0.1~15 htthp://ckan.org python-ckan python-ckanext-spatial python-carrot
+ python -m buildkit.deb . ckanext-csw 0.3~10 http://ckan.org python-ckanext-harvest python-owslib python-ckan
+ python -m buildkit.deb . ckanext-dgu 0.2~11 http://ckan.org python-ckan python-ckanext-importlib python-ckanext-dgu python-ckanext-csw python-ckan python-ckanext-spatial python-ckanext-inspire
+ python -m buildkit.deb . ckanext-qa 0.1~19 http://ckan.org python-ckan
+ python -m buildkit.deb . ckan 1.3.4~02 http://ckan.org python-routes python-vdm python-pylons python-genshi python-sqlalchemy python-repoze.who python-repoze.who-plugins python-pyutilib.component.core python-migrate python-formalchemy python-sphinx python-markupsafe python-setuptools python-psycopg2 python-licenses python-ckan-deps
There's a dependency on postfix. Choose internet site and the default hostname unless you know better.
@@ -201,7 +621,7 @@
::
- cd /var/packages/debian/
+ cd /var/packages/lucid/
sudo reprepro includedeb lucid ~/*.deb
You can remove them like this from the same directory:
@@ -210,211 +630,26 @@
sudo reprepro remove lucid python-ckan
-Testing in a VM
-===============
+Automatic Packaging
+===================
-If you aren't running Lucid, you may need to test in a VM. First set up a cache
-of the repositories so that you don't need to fetch packages each time you
-build a VM:
+The BuildKit script will build and place Debian packages in your ``missing``
+directory. Make sure there is nothing in there that shouldn't be overwritten by
+this script.
+
+To package everything automatically, run it like this:
::
- sudo apt-get install apt-proxy
+ cd missing
+ bin/python -m buildkit.update_all .
-Once this is complete, your (empty) proxy is ready for use on
-http://mirroraddress:9999 and will find Ubuntu repository under ``/ubuntu``.
+For each pacakge you'll be loaded into ``vim`` to edit the changelog. Save and
+quit when you are done. Names, version numbers and dependencies are
+automatically generated.
-See also:
+You should find all your packages nicely created now.
-* https://help.ubuntu.com/community/AptProxy
-
-Now create a directory ``~/Vms`` for your virtual machines.
-
-::
-
- mkdir ~/Vms
-
-
-We'll use manual bridging and networking rather than relying on the magic provided by ``libvirt``. Out virtual network for the VMs will be 192.168.100.xxx. You can use any number from 2-253 inclusive for the last bit of the IP. This first machine will have the IP address 192.168.100.2. Each virtual machine afterwards must have a unique IP address.
-
-First set some variables:
-
-::
-
- export THIS_IP="3"
- export HOST_IP="10.10.9.99"
-
-You can get the host IP by looking at the output from ``ifconifg``.
-
-Now create the VM:
-
-::
-
- cd ${HOME}/Vms
- export BASE_IP="192.168.100"
- sudo vmbuilder kvm ubuntu \
- --mem 512 \
- --cpus 4 \
- --domain ckan_${THIS_IP} \
- --dest ckan_${THIS_IP} \
- --flavour virtual \
- --suite lucid \
- --arch amd64 \
- --hostname ckan_${THIS_IP} \
- --user ubuntu \
- --pass ubuntu \
- --rootpass ubuntu \
- --debug -v \
- --ip ${BASE_IP}.${THIS_IP} \
- --mask 255.255.255.0 \
- --net ${BASE_IP}.0 \
- --bcast ${BASE_IP}.255 \
- --gw ${BASE_IP}.1 \
- --dns ${BASE_IP}.1 \
- --proxy http://${HOST_IP}:9999/ubuntu \
- --components main,universe \
- --addpkg vim \
- --addpkg openssh-server
-
-
-This assumes you already have an apt mirror set up on port 9999 as described
-above and that you are putting everything in ``~/Vms``.
-
-Now for the networking.
-
-First check you have forwarding enabled on the host:
-
-::
-
- sudo -s
- echo "1" > /proc/sys/net/ipv4/ip_forward
- exit
-
-Now save this as ``~/Vms/start.sh``:
-
-::
-
- #!/bin/bash
-
- if [ "X$1" = "X" ] || [ "X$2" = "X" ] || [ "X$3" = "X" ] || [ "X$4" = "X" ] || [ "X$5" = "X" ]; then
- echo "ERROR: call this script with network device name, tunnel name, amount of memory, number of CPUs and path to the image e.g."
- echo " $0 eth0 qtap0 512M 4 /home/Vms/ckan_2/tmpKfAdeU.qcow2 [extra args to KVM]"
- exit 1
- fi
-
- NETWORK_DEVICE=$1
- TUNNEL=$2
- MEM=$3
- CPUS=$4
- IMAGE=$5
- EXTRA=$6
- MACADDR="52:54:$(dd if=/dev/urandom count=1 2>/dev/null | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\).*$/\1:\2:\3:\4/')";
-
- echo "Creating bridge..."
- sudo iptables -t nat -A POSTROUTING -o ${NETWORK_DEVICE} -j MASQUERADE
- sudo brctl addbr br0
- sudo ifconfig br0 192.168.100.254 netmask 255.255.255.0 up
- echo "done."
- echo "Creating tunnel..."
- sudo modprobe tun
- sudo tunctl -b -u root -t ${TUNNEL}
- sudo brctl addif br0 ${TUNNEL}
- sudo ifconfig ${TUNNEL} up 0.0.0.0 promisc
- echo "done."
- echo "Starting VM ${IMAGE} on ${TUNNEL} via ${NETWORK_DEVICE} with MAC ${MACADDR}..."
- sudo /usr/bin/kvm -M pc-0.12 -enable-kvm -m ${MEM} -smp ${CPUS} -name dev -monitor pty -boot c -drive file=${IMAGE},if=ide,index=0,boot=on -net nic,macaddr=${MACADDR} -net tap,ifname=${TUNNEL},script=no,downscript=no -serial none -parallel none -usb ${EXTRA}
-
-Make it executable:
-
-::
-
- chmod a+x ~/Vms/start.sh
-
-Now you can start it:
-
-::
-
- ./start.sh eth1 qtap0 512M 1 /home/james/Vms/ckan_3/tmpuNIv2h.qcow2
-
-Now login:
-
-::
-
- ssh ubuntu@${BASE_IP}.${THIS_IP}
-
-Once in run this (swapping the repository name for the one you want to test):
-
-::
-
- sudo apt-get install wget
- wget -qO- http://apt-alpha.ckan.org/packages.okfn.key | sudo apt-key add -
- echo "deb http://apt-alpha.ckan.org/debian lucid universe" | sudo tee /etc/apt/sources.list.d/okfn.list
- sudo apt-get update
-
-If you change your host machine's networking you will probably need to update
-the ``/etc/resolv.conf`` in the guest.
-
-Now that yoy have the repo added you can install and test CKAN as normal.
-
-Relase Process
-==============
-
-For any instance of CKAN, the following release process occurs:
-
-* Package up all the ``.deb`` files in a directory with the release date in the
- format ``yyyy-mm-dd_nn`` where ``nn`` is the release the number of the
- release on each day. eg ``2011-03-13_01``
-
-* Import them into the dev repositoy:
-
- ::
-
- cd /var/packages/<instance>-dev
- sudo reprepro includedeb lucid /home/ubuntu/release/2011-03-13_01/*.deb
-
- Here's the pool of packages after the import:
-
- ::
-
- $ cd /var/packages/<instance>-dev
- $ find . | grep ".deb"
- ./pool/universe/p/python-apachemiddleware/python-apachemiddleware_0.1.0-1_amd64.deb
- ./pool/universe/p/python-ckan/python-ckan_1.3.2~10-1_amd64.deb
- ./pool/universe/p/python-ckanext-dgu/python-ckanext-dgu_0.2~06-1_amd64.deb
- ./pool/universe/p/python-licenses/python-licenses_0.6-1_amd64.deb
- ./pool/universe/p/python-ckanclient/python-ckanclient_0.6-1_amd64.deb
- ./pool/universe/p/python-vdm/python-vdm_0.9-1_amd64.deb
- ./pool/universe/p/python-ckan-deps/python-ckan-deps_1.3.4-1_amd64.deb
- ./pool/universe/p/python-owslib/python-owslib_0.3.2beta~02-1_amd64.deb
- ./pool/universe/p/python-formalchemy/python-formalchemy_1.3.6-1_amd64.deb
- ./pool/universe/p/python-solrpy/python-solrpy_0.9.3-1_amd64.deb
- ./pool/universe/p/python-markupsafe/python-markupsafe_0.9.2-1_amd64.deb
- ./pool/universe/p/python-ckanext-qa/python-ckanext-qa_0.1~09-1_amd64.deb
- ./pool/universe/p/python-ckanext-csw/python-ckanext-csw_0.3~04-1_amd64.deb
- ./pool/universe/p/python-pyutilib.component.core/python-pyutilib.component.core_4.1-1_amd64.deb
- ./pool/universe/c/ckan/ckan_1.3.2~09_amd64.deb
- ./pool/universe/c/ckan-dgu/ckan-dgu_0.2~05_amd64.deb
-
-* Test on the dev server, if everything is OK, copy the dev repo to UAT:
-
- ::
-
- $ cd /var/packages/
- $ sudo rm -r <instance>-uat
- $ sudo cp -pr <instance>-dev <instance>-uat
-
-* You can now run this on UAT:
-
- ::
-
- sudo apt-get update
- sudo apt-get upgrade
-
- Because it is an exact copy of the dev repo at the point you tested you can
- be sure the software is the same
-
-* If all goes well, repeat this process with staging and live repos to deploy the release.
-
Next Steps
==========
@@ -422,4 +657,239 @@
* Delayed updates
+Proposed Changes to CKAN
+========================
+* Change the config file to support file based logging by default
+* Move who.ini into the config
+* Add a ckan/wsgi.py for standard DGU deployment
+* Modify __init__.py to change
+
+
+* No __init__.py in test directory
+
+
+
+ubuntu at ip-10-226-226-132:/var/packages/dgu-uat$ sudo reprepro includedeb lucid /home/ubuntu/release/2011-04-18_01/*.deb
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.2~10_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.2~10' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.2~11_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.2~11' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.4~01_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.4~01' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.4~02_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.4~02' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.4~03_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.4~03' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~06_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~06' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~07_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~07' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~08_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~08' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~09_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~09' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~11_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~11' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~12_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~12' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~13_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~13' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~14_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~14' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~15_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~15' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/python-ckan_1.3.4~01-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckan' '1.3.4~01-1' in 'lucid|universe|amd64', as it has already '1.3.4~02-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckan_1.3.4~02-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckan' '1.3.4~02-1' in 'lucid|universe|amd64', as it has already '1.3.4~02-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-csw_0.3~10-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-csw' '0.3~10-1' in 'lucid|universe|amd64', as it has already '0.3~10-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-dgu_0.2~08-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-dgu' '0.2~08-1' in 'lucid|universe|amd64', as it has already '0.2~11-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-dgu_0.2~09-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-dgu' '0.2~09-1' in 'lucid|universe|amd64', as it has already '0.2~11-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-dgu_0.2~10-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-dgu' '0.2~10-1' in 'lucid|universe|amd64', as it has already '0.2~11-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-dgu_0.2~11-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-dgu' '0.2~11-1' in 'lucid|universe|amd64', as it has already '0.2~11-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-harvest_0.1~13-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-harvest' '0.1~13-1' in 'lucid|universe|amd64', as it has already '0.1~15-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-harvest_0.1~14-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-harvest' '0.1~14-1' in 'lucid|universe|amd64', as it has already '0.1~15-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-harvest_0.1~15-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-harvest' '0.1~15-1' in 'lucid|universe|amd64', as it has already '0.1~15-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-inspire_0.1~01-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-inspire' '0.1~01-1' in 'lucid|universe|amd64', as it has already '0.1~03-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-inspire_0.1~02-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-inspire' '0.1~02-1' in 'lucid|universe|amd64', as it has already '0.1~03-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-inspire_0.1~03-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-inspire' '0.1~03-1' in 'lucid|universe|amd64', as it has already '0.1~03-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-qa_0.1~19-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-qa' '0.1~19-1' in 'lucid|universe|amd64', as it has already '0.1~19-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-spatial_0.1~01-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-spatial' '0.1~01-1' in 'lucid|universe|amd64', as it has already '0.1~04-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-spatial_0.1~03-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-spatial' '0.1~03-1' in 'lucid|universe|amd64', as it has already '0.1~04-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-spatial_0.1~04-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-spatial' '0.1~04-1' in 'lucid|universe|amd64', as it has already '0.1~04-1'.
+/home/ubuntu/release/2011-04-18_01/python-owslib_0.3.2beta~03-1_amd64.deb: component guessed as 'universe'
+ERROR: '/home/ubuntu/release/2011-04-18_01/python-owslib_0.3.2beta~03-1_amd64.deb' cannot be included as 'pool/universe/p/python-owslib/python-owslib_0.3.2beta~03-1_amd64.deb'.
+Already existing files can only be included again, if they are the same, but:
+md5 expected: 3f38d2e844c8d6ec15da6ba51910f3e2, got: ee48427eb11f8152f50f6dc93aeb70d4
+sha1 expected: 87cd7724d8d8f0aaeaa24633abd86e02297771d7, got: 8476b1b0e022892ceb8a35f1848818c31d7441bf
+sha256 expected: 4c9937c78be05dfa5b9dfc85f3a26a51ca4ec0a2d44e8bca530a0c85f12ef400, got: ad3f7458d069a9dd268d144577a7932735643056e45d0a30b7460c38e64057d7
+size expected: 57658, got: 57656
+/home/ubuntu/release/2011-04-18_01/python-owslib_0.3.2beta~04-1_amd64.deb: component guessed as 'universe'
+Exporting indices...
+19A05DDEB16777A2 James Gardner (thejimmyg) <james.gardner at okfn.org> needs a passphrase
+Please enter passphrase:
+Deleting files just added to the pool but not used (to avoid use --keepunusednewfiles next time)
+deleting and forgetting pool/universe/c/ckan/ckan_1.3.2~10_amd64.deb
+deleting and forgetting pool/universe/c/ckan/ckan_1.3.2~11_amd64.deb
+deleting and forgetting pool/universe/c/ckan/ckan_1.3.4~01_amd64.deb
+deleting and forgetting pool/universe/c/ckan/ckan_1.3.4~02_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~06_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~07_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~08_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~09_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~11_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~12_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~13_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~14_amd64.deb
+deleting and forgetting pool/universe/p/python-ckan/python-ckan_1.3.4~01-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-dgu/python-ckanext-dgu_0.2~08-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-dgu/python-ckanext-dgu_0.2~09-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-dgu/python-ckanext-dgu_0.2~10-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-harvest/python-ckanext-harvest_0.1~13-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-harvest/python-ckanext-harvest_0.1~14-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-inspire/python-ckanext-inspire_0.1~01-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-inspire/python-ckanext-inspire_0.1~02-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-spatial/python-ckanext-spatial_0.1~01-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-spatial/python-ckanext-spatial_0.1~03-1_amd64.deb
+Not deleting possibly left over files due to previous errors.
+(To keep the files in the still existing index files from vanishing)
+Use dumpunreferenced/deleteunreferenced to show/delete files without references.
+1 files lost their last reference.
+(dumpunreferenced lists such files, use deleteunreferenced to delete them.)
+There have been errors!
+(reverse-i-search)`delete': cat src/pip-^Clete-this-directory.txt
+ubuntu at ip-10-226-226-132:/var/packages/dgu-uat$ sudo reprepro deleteunreferenced --help
+Error: Too many arguments for command 'deleteunreferenced'!
+Syntax: reprepro deleteunreferenced
+There have been errors!
+ubuntu at ip-10-226-226-132:/var/packages/dgu-uat$ sudo reprepro deleteunreferenced
+deleting and forgetting pool/universe/p/python-owslib/python-owslib_0.3.2beta~03-1_amd64.deb
+ubuntu at ip-10-226-226-132:/var/packages/dgu-uat$
+ubuntu at ip-10-226-226-132:/var/packages/dgu-uat$ sudo reprepro includedeb lucid /home/ubuntu/release/2011-04-18_01/*.deb
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.2~10_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.2~10' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.2~11_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.2~11' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.4~01_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.4~01' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.4~02_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.4~02' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan_1.3.4~03_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan' '1.3.4~03' in 'lucid|universe|amd64', as it has already '1.3.4~03'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~06_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~06' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~07_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~07' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~08_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~08' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~09_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~09' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~11_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~11' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~12_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~12' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~13_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~13' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~14_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~14' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/ckan-dgu_0.2~15_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'ckan-dgu' '0.2~15' in 'lucid|universe|amd64', as it has already '0.2~15'.
+/home/ubuntu/release/2011-04-18_01/python-ckan_1.3.4~01-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckan' '1.3.4~01-1' in 'lucid|universe|amd64', as it has already '1.3.4~02-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckan_1.3.4~02-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckan' '1.3.4~02-1' in 'lucid|universe|amd64', as it has already '1.3.4~02-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-csw_0.3~10-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-csw' '0.3~10-1' in 'lucid|universe|amd64', as it has already '0.3~10-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-dgu_0.2~08-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-dgu' '0.2~08-1' in 'lucid|universe|amd64', as it has already '0.2~11-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-dgu_0.2~09-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-dgu' '0.2~09-1' in 'lucid|universe|amd64', as it has already '0.2~11-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-dgu_0.2~10-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-dgu' '0.2~10-1' in 'lucid|universe|amd64', as it has already '0.2~11-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-dgu_0.2~11-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-dgu' '0.2~11-1' in 'lucid|universe|amd64', as it has already '0.2~11-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-harvest_0.1~13-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-harvest' '0.1~13-1' in 'lucid|universe|amd64', as it has already '0.1~15-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-harvest_0.1~14-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-harvest' '0.1~14-1' in 'lucid|universe|amd64', as it has already '0.1~15-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-harvest_0.1~15-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-harvest' '0.1~15-1' in 'lucid|universe|amd64', as it has already '0.1~15-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-inspire_0.1~01-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-inspire' '0.1~01-1' in 'lucid|universe|amd64', as it has already '0.1~03-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-inspire_0.1~02-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-inspire' '0.1~02-1' in 'lucid|universe|amd64', as it has already '0.1~03-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-inspire_0.1~03-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-inspire' '0.1~03-1' in 'lucid|universe|amd64', as it has already '0.1~03-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-qa_0.1~19-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-qa' '0.1~19-1' in 'lucid|universe|amd64', as it has already '0.1~19-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-spatial_0.1~01-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-spatial' '0.1~01-1' in 'lucid|universe|amd64', as it has already '0.1~04-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-spatial_0.1~03-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-spatial' '0.1~03-1' in 'lucid|universe|amd64', as it has already '0.1~04-1'.
+/home/ubuntu/release/2011-04-18_01/python-ckanext-spatial_0.1~04-1_amd64.deb: component guessed as 'universe'
+Skipping inclusion of 'python-ckanext-spatial' '0.1~04-1' in 'lucid|universe|amd64', as it has already '0.1~04-1'.
+/home/ubuntu/release/2011-04-18_01/python-owslib_0.3.2beta~03-1_amd64.deb: component guessed as 'universe'
+ERROR: '/home/ubuntu/release/2011-04-18_01/python-owslib_0.3.2beta~03-1_amd64.deb' cannot be included as 'pool/universe/p/python-owslib/python-owslib_0.3.2beta~03-1_amd64.deb'.
+Already existing files can only be included again, if they are the same, but:
+md5 expected: 3f38d2e844c8d6ec15da6ba51910f3e2, got: ee48427eb11f8152f50f6dc93aeb70d4
+sha1 expected: 87cd7724d8d8f0aaeaa24633abd86e02297771d7, got: 8476b1b0e022892ceb8a35f1848818c31d7441bf
+sha256 expected: 4c9937c78be05dfa5b9dfc85f3a26a51ca4ec0a2d44e8bca530a0c85f12ef400, got: ad3f7458d069a9dd268d144577a7932735643056e45d0a30b7460c38e64057d7
+size expected: 57658, got: 57656
+/home/ubuntu/release/2011-04-18_01/python-owslib_0.3.2beta~04-1_amd64.deb: component guessed as 'universe'
+Exporting indices...
+19A05DDEB16777A2 James Gardner (thejimmyg) <james.gardner at okfn.org> needs a passphrase
+Please enter passphrase:
+Deleting files just added to the pool but not used (to avoid use --keepunusednewfiles next time)
+deleting and forgetting pool/universe/c/ckan/ckan_1.3.2~10_amd64.deb
+deleting and forgetting pool/universe/c/ckan/ckan_1.3.2~11_amd64.deb
+deleting and forgetting pool/universe/c/ckan/ckan_1.3.4~01_amd64.deb
+deleting and forgetting pool/universe/c/ckan/ckan_1.3.4~02_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~06_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~07_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~08_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~09_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~11_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~12_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~13_amd64.deb
+deleting and forgetting pool/universe/c/ckan-dgu/ckan-dgu_0.2~14_amd64.deb
+deleting and forgetting pool/universe/p/python-ckan/python-ckan_1.3.4~01-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-dgu/python-ckanext-dgu_0.2~08-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-dgu/python-ckanext-dgu_0.2~09-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-dgu/python-ckanext-dgu_0.2~10-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-harvest/python-ckanext-harvest_0.1~13-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-harvest/python-ckanext-harvest_0.1~14-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-inspire/python-ckanext-inspire_0.1~01-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-inspire/python-ckanext-inspire_0.1~02-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-spatial/python-ckanext-spatial_0.1~01-1_amd64.deb
+deleting and forgetting pool/universe/p/python-ckanext-spatial/python-ckanext-spatial_0.1~03-1_amd64.deb
+Not deleting possibly left over files due to previous errors.
+(To keep the files in the still existing index files from vanishing)
+Use dumpunreferenced/deleteunreferenced to show/delete files without references.
+1 files lost their last reference.
+(dumpunreferenced lists such files, use deleteunreferenced to delete them.)
+There have been errors!
+(reverse-i-search)`delete': cat src/pip-^Clete-this-directory.txt
+ubuntu at ip-10-226-226-132:/var/packages/dgu-uat$ sudo reprepro deleteunreferenced --help
+Error: Too many arguments for command 'deleteunreferenced'!
+Syntax: reprepro deleteunreferenced
+There have been errors!
+ubuntu at ip-10-226-226-132:/var/packages/dgu-uat$ sudo reprepro deleteunreferenced
+deleting and forgetting pool/universe/p/python-owslib/python-owslib_0.3.2beta~03-1_amd64.deb
+ubuntu at ip-10-226-226-132:/var/packages/dgu-uat$
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/doc/vm.rst Tue May 10 11:27:25 2011 +0100
@@ -0,0 +1,186 @@
+Testing CKAN in a VM
+++++++++++++++++++++
+
+WARNING: This document is still under development, use only if you are a member
+of the CKAN team who wishes to be an early adopter and are interested in
+experimenting with virtual machines.
+
+If you aren't running Lucid, you may need to test in a VM. First set up a cache
+of the repositories so that you don't need to fetch packages each time you
+build a VM:
+
+::
+
+ sudo apt-get install apt-proxy
+
+Once this is complete, your (empty) proxy is ready for use on
+http://mirroraddress:9999 and will find Ubuntu repository under ``/ubuntu``.
+
+See also:
+
+* https://help.ubuntu.com/community/AptProxy
+
+Now create a directory ``~/Vms`` for your virtual machines.
+
+::
+
+ mkdir ~/Vms
+
+
+We'll use manual bridging and networking rather than relying on the magic provided by ``libvirt``. Out virtual network for the VMs will be 192.168.100.xxx. You can use any number from 2-253 inclusive for the last bit of the IP. This first machine will have the IP address 192.168.100.2. Each virtual machine afterwards must have a unique IP address.
+
+First set some variables:
+
+::
+
+ export THIS_IP="4"
+ export HOST_IP="192.168.0.2"
+
+You can get the host IP by looking at the output from ``ifconifg``.
+
+Now create the VM:
+
+::
+
+ cd ${HOME}/Vms
+ export BASE_IP="192.168.100"
+ sudo vmbuilder kvm ubuntu \
+ --mem 512 \
+ --cpus 4 \
+ --domain ckan_${THIS_IP} \
+ --dest ckan_${THIS_IP} \
+ --flavour virtual \
+ --suite lucid \
+ --arch amd64 \
+ --hostname ckan${THIS_IP} \
+ --user ubuntu \
+ --pass ubuntu \
+ --rootpass ubuntu \
+ --debug -v \
+ --ip ${BASE_IP}.${THIS_IP} \
+ --mask 255.255.255.0 \
+ --net ${BASE_IP}.0 \
+ --bcast ${BASE_IP}.255 \
+ --gw ${BASE_IP}.254 \
+ --dns ${BASE_IP}.254 \
+ --proxy http://${HOST_IP}:9999/ubuntu \
+ --components main,universe \
+ --addpkg vim \
+ --addpkg openssh-server \
+ --addpkg wget
+
+This assumes you already have an apt mirror set up on port 9999 as described
+above and that you are putting everything in ``~/Vms``.
+
+Now for the networking.
+
+First check you have forwarding enabled on the host:
+
+::
+
+ sudo -s
+ echo "1" > /proc/sys/net/ipv4/ip_forward
+ exit
+
+Now save this as ``~/Vms/start.sh``:
+
+::
+
+ #!/bin/bash
+
+ if [ "X$1" = "X" ] || [ "X$2" = "X" ] || [ "X$3" = "X" ] || [ "X$4" = "X" ] || [ "X$5" = "X" ]; then
+ echo "ERROR: call this script with network device name, tunnel name, amount of memory, number of CPUs and path to the image e.g."
+ echo " $0 eth0 qtap0 512M 4 /home/Vms/ckan_2/tmpKfAdeU.qcow2 [extra args to KVM]"
+ exit 1
+ fi
+
+ NETWORK_DEVICE=$1
+ TUNNEL=$2
+ MEM=$3
+ CPUS=$4
+ IMAGE=$5
+ EXTRA=$6
+ MACADDR="52:54:$(dd if=/dev/urandom count=1 2>/dev/null | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\).*$/\1:\2:\3:\4/')";
+
+ echo "Creating bridge..."
+ sudo iptables -t nat -A POSTROUTING -o ${NETWORK_DEVICE} -j MASQUERADE
+ sudo brctl addbr br0
+ sudo ifconfig br0 192.168.100.254 netmask 255.255.255.0 up
+ echo "done."
+ echo "Creating tunnel..."
+ sudo modprobe tun
+ sudo tunctl -b -u root -t ${TUNNEL}
+ sudo brctl addif br0 ${TUNNEL}
+ sudo ifconfig ${TUNNEL} up 0.0.0.0 promisc
+ echo "done."
+ echo "Starting VM ${IMAGE} on ${TUNNEL} via ${NETWORK_DEVICE} with MAC ${MACADDR}..."
+ sudo /usr/bin/kvm -M pc-0.12 -enable-kvm -m ${MEM} -smp ${CPUS} -name dev -monitor pty -boot c -drive file=${IMAGE},if=ide,index=0,boot=on -net nic,macaddr=${MACADDR} -net tap,ifname=${TUNNEL},script=no,downscript=no -serial none -parallel none -usb ${EXTRA}
+
+Make it executable:
+
+::
+
+ chmod a+x ~/Vms/start.sh
+
+Now you can start it:
+
+::
+
+ ./start.sh eth1 qtap0 512M 1 /home/james/Vms/ckan_3/tmpuNIv2h.qcow2
+
+Now login:
+
+::
+
+ ssh ubuntu@${BASE_IP}.${THIS_IP}
+
+Once in you'll need some more configuration.
+
+Edit ``/etc/resolv.conf`` to contain just this (the Google DNS servers, handy
+to used a fixed IP so that you don't have to update your ``resolve.conf`` each
+time you move to a different network):
+
+::
+
+ nameserver 8.8.8.8
+
+Then change ``/etc/apt/apt.conf`` to comment out the proxy line, you may as
+well get updates directly now.
+
+Finally, run this (swapping the repository name for the one you want to test)
+to allow yourself to install CKAN:
+
+::
+
+ sudo apt-get install wget
+ wget -qO- http://apt-alpha.ckan.org/packages.okfn.key | sudo apt-key add -
+ echo "deb http://apt-alpha.ckan.org/debian lucid universe" | sudo tee /etc/apt/sources.list.d/okfn.list
+ sudo apt-get update
+
+Now that you have the repo added you can install and test CKAN as normal.
+
+Here's how mine look:
+
+::
+
+ ubuntu at ckan4:~$ cat /etc/network/interfaces
+ # This file describes the network interfaces available on your system
+ # and how to activate them. For more information, see interfaces(5).
+
+ # The loopback network interface
+ auto lo
+ iface lo inet loopback
+
+ # The primary network interface
+ auto eth0
+ iface eth0 inet static
+ address 192.168.100.4
+ netmask 255.255.255.0
+ network 192.168.100.0
+ broadcast 192.168.100.255
+ gateway 192.168.100.254
+ dns 192.168.100.254
+ ubuntu at ckan4:~$ cat /etc/resolv.conf
+ nameserver 8.8.8.8
+
+
Repository URL: https://bitbucket.org/okfn/ckan/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list