[ckan-changes] commit/ckan: 2 new changesets
Bitbucket
commits-noreply at bitbucket.org
Fri Nov 4 15:58:35 UTC 2011
2 new commits in ckan:
https://bitbucket.org/okfn/ckan/changeset/b3b54069ae25/
changeset: b3b54069ae25
branch: release-v1.5
user: thejimmyg
date: 2011-11-04 16:54:37
summary: [packaging] Changes to get the packaging working with buildkit 0.2.0
affected #: 10 files
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 .hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -8,6 +8,7 @@
sandbox/*
theme/*
theme
+dist
# pylons
development.ini*
@@ -33,3 +34,9 @@
# nosetest coverage output
.coverage
htmlcov/*
+
+# CKAN
+ckan.log
+ckan_deb/DEBIAN/control
+ckan_deb/DEBIAN/prerm
+
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 MANIFEST.in
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,6 +2,7 @@
recursive-include ckan/public *
recursive-include ckan/config *.ini
recursive-include ckan/config *.xml
+recursive-include ckan/i18n *
recursive-include ckan/templates *
recursive-include ckan *.ini
prune .hg
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 build.sh
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# This is a script which uses BuildKit 0.2.0 to automatically package CKAN and
+# its dependencies for use as Debian packages on Ubuntu 10.04.
+#
+# See the BuildKit documentation or look at the help to see what these commands
+# do. eg:
+#
+# buildkit --help
+#
+# The licenses and solrpy packages have to be built separately from the others
+# because they are slighlty non-standard. All other CKAN dependencies are
+# automatically handled by the --deps option to the CKAN build command.
+#
+# Over time, as CKAN and its dependencies follow conventions more closely the
+# edge cases handled by this script should go away and the build should become
+# simpler.
+
+CKAN_PACKAGE_VERSION=08
+DEPS_PACKAGE_VERSION=05
+CKAN_PATH=/home/james/Documents/Work/OKFN/code/pyenv/src/ckan
+BUILDKIT_REPO_PATH="/var/lib/buildkit/repo/ckan-1.5"
+PIP_DOWNLOAD_CACHE=${CKAN_PATH}/build/env/cache
+EMAIL=packaging at okfn.org
+NAME="James Gardner"
+
+# Clean the build environment
+echo "Cleaning the environment ..."
+rm -r ${CKAN_PATH}/dist/
+rm -rf ${CKAN_PATH}/build/env/src
+mkdir -p ${CKAN_PATH}/dist/buildkit
+echo "done."
+
+echo "Buildling the packages ..."
+# Create the python-ckan debian package
+buildkit pkg python -p $CKAN_PACKAGE_VERSION --delete "solrpy" --distro-dep "python-solr" --delete "licenses" --distro-dep "python-licenses" --delete "repoze.who-friendlyform" --rename "repoze.who.plugins.openid -> repoze.who-plugins" --rename "babel -> pybabel" --author-email="$EMAIL" --author-name="$NAME" --packager-email="$EMAIL" --packager-name="$NAME" --deps --exclude=test/generate_package --conflict-module "sqlalchemy-migrate -> migrate" --conflict-module "sqlalchemy -> lib/sqlalchemy" --debian-dir --url http://ckan.org ${CKAN_PATH}
+
+# Creates the ckan debian package (of which python-ckan is a dependency)
+buildkit pkg nonpython -p $CKAN_PACKAGE_VERSION --deb --output-dir ${CKAN_PATH}/dist/buildkit ${CKAN_PATH}/ckan_deb
+
+# Build python-solr
+${CKAN_PATH}/build/buildkit/env/bin/pip install --download-cache ${CKAN_PATH}/build/buildkit/env/cache --no-install --upgrade "solrpy==0.9.4"
+mv ${CKAN_PATH}/build/buildkit/env/build/solrpy ${CKAN_PATH}/build/buildkit/env/build/solr
+# We need to rename the package here
+sed -e "s,solrpy,solr," -i ${CKAN_PATH}/build/buildkit/env/build/solr/setup.py
+# We need to specify an author explicitly since it is missing we'll use the CKAN one
+buildkit pkg python -p $DEPS_PACKAGE_VERSION --author-email="$EMAIL" --author-name="$NAME" --packager-email="$EMAIL" --packager-name="$NAME" --debian-dir ${CKAN_PATH}/build/buildkit/env/build/solr/
+cp ${CKAN_PATH}/build/buildkit/env/build/solr/dist/buildkit/*.deb ${CKAN_PATH}/dist/buildkit/
+
+# Build python-licenses
+${CKAN_PATH}/build/buildkit/env/bin/pip install --download-cache ${CKAN_PATH}/build/buildkit/env/cache --no-install --upgrade "licenses==0.6.1"
+buildkit pkg python --deb -p $DEPS_PACKAGE_VERSION --author-email="$EMAIL" --author-name="$NAME" --packager-email="$EMAIL" --packager-name="$NAME" ${CKAN_PATH}/build/buildkit/env/build/licenses
+cp ${CKAN_PATH}/build/buildkit/env/build/licenses/dist/buildkit/*.deb ${CKAN_PATH}/dist/buildkit/
+echo "done."
+
+# Add the .debs to the repository and the export the latest files for upload
+echo "Adding the packages to the $BUILDKIT_REPO_PATH repo ..."
+sudo -u buildkit buildkit repo remove -a $BUILDKIT_REPO_PATH
+sudo -u buildkit buildkit repo add $BUILDKIT_REPO_PATH ${CKAN_PATH}/dist/buildkit/*.deb
+echo "done."
+
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 ckan/i18n/__init__.py
--- a/ckan/i18n/__init__.py
+++ b/ckan/i18n/__init__.py
@@ -0,0 +1,1 @@
+# Need some content here to avoid the packaging stripping it out
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 ckan_deb/DEBIAN/control.template
--- a/ckan_deb/DEBIAN/control.template
+++ b/ckan_deb/DEBIAN/control.template
@@ -5,7 +5,7 @@
Maintainer: James Gardner <james.gardner at okfn.org>
Section: main/web
Priority: extra
-Depends: python-ckan, postgresql-8.4, apache2, libapache2-mod-wsgi, python-virtualenv, python-pip, solr-jetty
+Depends: python-ckan, apache2, libapache2-mod-wsgi, python-virtualenv, python-pip
Homepage: http://ckan.org
Description: ckan
Common files useful for managing CKAN installations
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 ckan_deb/usr/bin/ckan-create-instance
--- a/ckan_deb/usr/bin/ckan-create-instance
+++ b/ckan_deb/usr/bin/ckan-create-instance
@@ -1,7 +1,6 @@
#!/bin/bash
. /usr/lib/ckan/common.sh
-INSTANCE=$1
# Check we are root
if [[ $EUID -ne 0 ]]; then
@@ -9,6 +8,25 @@
exit 1
fi
+if [ $# -lt 3 ]
+then
+ echo "ERROR: Expected two arguments (instance name, hostname, local database [yes or no]) eg."
+ echo " $0 std std.ckan.org yes"
+ exit 1
+fi
+
+INSTANCE=$1
+HOSTNAME=$2
+LOCAL_DB=$3
+
+# Check the LOCAL_DB variable
+
+if ! [[ ( "$LOCAL_DB" == "yes" ) || ("$LOCAL_DB" == "no" ) ]]
+then
+ echo "Expceted the LOCAL_DB variable to be 'yes' or 'no', not '$LOCAL_DB'"
+ exit 1
+fi
+
# Create an install settings file if it doesn't exist
if ! [ -f /etc/ckan/${INSTANCE}/install_settings.sh ] ; then
mkdir -p /etc/ckan/${INSTANCE}/
@@ -37,9 +55,6 @@
echo "Installing or upgrading CKAN ${INSTANCE} ..."
-echo "Making sure PostgreSQL is running ..."
-/etc/init.d/postgresql-8.4 start
-
echo "Ensuring users and groups are set up correctly ..."
ckan_ensure_users_and_groups ${INSTANCE}
@@ -73,36 +88,34 @@
echo "Ensuring wsgi.py file exists ..."
ckan_create_wsgi_handler ${INSTANCE}
-# Replace any existing user with a new one with this password
-echo "Setting the password of the ${INSTANCE} user in PostgreSQL"
-ckan_add_or_replace_database_user ${INSTANCE} ${CKAN_DB_PASSWORD}
+if [[ ( "$LOCAL_DB" == "yes" ) ]]
+then
+ # Replace any existing user with a new one with this password
+ echo "Making sure PostgreSQL is running ..."
+ /etc/init.d/postgresql-8.4 start
+
+ echo "Setting the password of the ${INSTANCE} user in PostgreSQL"
+ ckan_add_or_replace_database_user ${INSTANCE} ${CKAN_DB_PASSWORD}
+fi
-# Solr support
-echo "Setting up Solr ..."
-sed \
- -e "s,NO_START=1,NO_START=0," \
- -e "s,#JETTY_HOST=\$(uname -n),JETTY_HOST=127.0.0.1," \
- -e "s,#JETTY_PORT=8080,JETTY_PORT=8983," \
- -i /etc/default/jetty
-mv /usr/share/solr/conf/schema.xml /usr/share/solr/conf/schema.xml.`date --utc "+%Y-%m-%d_%T"`.bak
-ln -s /usr/lib/pymodules/python2.6/ckan/config/schema.xml /usr/share/solr/conf/schema.xml
-service jetty start
+if ! [ -f /etc/ckan/${INSTANCE}/${INSTANCE}.ini ] ; then
+ # Create the config file
+ echo "Creating/overwriting the config for CKAN ... "
+ ckan_create_config_file ${INSTANCE} ${CKAN_DB_PASSWORD} ${LOCAL_DB}
+ # Now that the file exists, make some customisations
+ sed \
+ -e "s,^\(ckan.dump_dir\)[ =].*,\1 = /var/lib/ckan/${INSTANCE}/static/dump," \
+ -i /etc/ckan/${INSTANCE}/${INSTANCE}.ini
+ #echo "Ensuring the latest plugins are configured ..."
+ #sed -e "s/^\(ckan.plugins\)[ =].*/\1 = ${INSTANCE}/" \
+ # -i /etc/ckan/${INSTANCE}/${INSTANCE}.ini
+fi
-# Create the config file
-echo "Creating/overwriting the config for CKAN ... "
-# We use the actual password in PostgreSQL in case any non-sense has gone on
-ckan_create_config_file ${INSTANCE} ${CKAN_DB_PASSWORD}
-# Now that the file exists, make some customisations
-sed \
- -e "s,^\(ckan.dump_dir\)[ =].*,\1 = /var/lib/ckan/${INSTANCE}/static/dump," \
- -i /etc/ckan/${INSTANCE}/${INSTANCE}.ini
-
-echo "Ensuring the ${INSTANCE} database exists ..."
-ckan_ensure_db_exists ${INSTANCE}
-
-#echo "Ensuring the latest plugins are configured ..."
-#sed -e "s/^\(ckan.plugins\)[ =].*/\1 = ${INSTANCE}/" \
-# -i /etc/ckan/${INSTANCE}/${INSTANCE}.ini
+if [[ ( "$LOCAL_DB" == "yes" ) ]]
+then
+ echo "Ensuring the ${INSTANCE} database exists ..."
+ ckan_ensure_db_exists ${INSTANCE}
+fi
# Overwrite the existing Apache config
if [ -f /etc/apache2/sites-enabled/000-default ] ; then
@@ -111,7 +124,7 @@
fi
echo "Overwriting the existing Apache config ..."
-ckan_overwrite_apache_config ${INSTANCE} ${INSTANCE}.ckan.localhost
+ckan_overwrite_apache_config ${INSTANCE} ${HOSTNAME}
# Make sure mod_rewrite is enabled
if ! [ -f /etc/apache2/mods-enabled/rewrite.load ] ; then
@@ -119,9 +132,12 @@
a2enmod rewrite
fi
-# Standard paster db upgrade
-echo "Performing any database upgrades ..."
-paster --plugin=ckan db upgrade --config=/etc/ckan/${INSTANCE}/${INSTANCE}.ini
+if [[ ( "$LOCAL_DB" == "yes" ) ]]
+then
+ # Standard paster db upgrade
+ echo "Performing any database upgrades ..."
+ paster --plugin=ckan db upgrade --config=/etc/ckan/${INSTANCE}/${INSTANCE}.ini
+fi
# Make sure our INSTANCE is enabled
echo "Bringing the ${INSTANCE} INSTANCE out of maintenance mode ..."
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 ckan_deb/usr/bin/ckan-setup-solr
--- /dev/null
+++ b/ckan_deb/usr/bin/ckan-setup-solr
@@ -0,0 +1,27 @@
+#!/bin/bash
+. /usr/lib/ckan/common.sh
+
+# Check we are root
+if [[ $EUID -ne 0 ]]; then
+ echo "This script must be run as root"
+ exit 1
+fi
+
+# Install solr-jetty if needed
+if ! [ -f /etc/default/jetty ] ; then
+ echo "Installing solr-jetty ..."
+ apt-get update
+ apt-get install -y solr-jetty
+ echo "done."
+fi
+
+echo "Configuring Solr for use with CKAN ..."
+sed \
+ -e "s,NO_START=1,NO_START=0," \
+ -e "s,#JETTY_HOST=\$(uname -n),JETTY_HOST=127.0.0.1," \
+ -e "s,#JETTY_PORT=8080,JETTY_PORT=8983," \
+ -i /etc/default/jetty
+mv /usr/share/solr/conf/schema.xml /usr/share/solr/conf/schema.xml.`date --utc "+%Y-%m-%d_%T"`.bak
+ln -s /usr/lib/pymodules/python2.6/ckan/config/schema.xml /usr/share/solr/conf/schema.xml
+service jetty start
+echo "done."
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 ckan_deb/usr/lib/ckan/common.sh
--- a/ckan_deb/usr/lib/ckan/common.sh
+++ b/ckan_deb/usr/lib/ckan/common.sh
@@ -90,7 +90,7 @@
}
ckan_create_config_file () {
- local INSTANCE password
+ local INSTANCE password LOCAL_DB
if [ "X$1" = "X" ] || [ "X$2" = "X" ] ; then
echo "ERROR: call the function create_config_file function with an INSTANCE name, and a password for postgresql e.g."
echo " dgu 1U923hjkh8"
@@ -98,19 +98,23 @@
else
INSTANCE=$1
password=$2
+ LOCAL_DB=$3
# Create an install settings file if it doesn't exist
if [ -f /etc/ckan/${INSTANCE}/${INSTANCE}.ini ] ; then
mv /etc/ckan/${INSTANCE}/${INSTANCE}.ini "/etc/ckan/${INSTANCE}/${INSTANCE}.ini.`date +%F_%T`.bak"
fi
paster make-config ckan /etc/ckan/${INSTANCE}/${INSTANCE}.ini
+
+ if [[ ( "$LOCAL_DB" == "yes" ) ]]
+ then
+ sed -e "s,^\(sqlalchemy.url\)[ =].*,\1 = postgresql://${INSTANCE}:${password}@localhost/${INSTANCE}," \
+ -i /etc/ckan/${INSTANCE}/${INSTANCE}.ini
+ fi
sed -e "s,^\(email_to\)[ =].*,\1 = root," \
-e "s,^\(error_email_from\)[ =].*,\1 = ckan-${INSTANCE}@`hostname`," \
-e "s,^\(cache_dir\)[ =].*,\1 = /var/lib/ckan/${INSTANCE}/data," \
-e "s,^\(who\.config_file\)[ =].*,\1 = /etc/ckan/${INSTANCE}/who.ini," \
- -e "s,^\(sqlalchemy.url\)[ =].*,\1 = postgresql://${INSTANCE}:${password}@localhost/${INSTANCE}," \
-e "s,ckan\.log,/var/log/ckan/${INSTANCE}/${INSTANCE}.log," \
- -e "s,ckan\.site_id,${INSTANCE}," \
- -e "s,ckan\.site_description,${INSTANCE}," \
-e "s,#solr_url = http://127.0.0.1:8983/solr,solr_url = http://127.0.0.1:8983/solr," \
-i /etc/ckan/${INSTANCE}/${INSTANCE}.ini
sudo chown ckan${INSTANCE}:ckan${INSTANCE} /etc/ckan/${INSTANCE}/${INSTANCE}.ini
@@ -169,8 +173,12 @@
sudo mkdir /var/lib/ckan/${INSTANCE}/pyenv
sudo chown -R ckan${INSTANCE}:ckan${INSTANCE} /var/lib/ckan/${INSTANCE}/pyenv
sudo -u ckan${INSTANCE} virtualenv --setuptools /var/lib/ckan/${INSTANCE}/pyenv
- echo "Attempting to install Pip 1.0 from pypi.python.org into pyenv to be used for extensions ..."
- sudo -u ckan${INSTANCE} /var/lib/ckan/${INSTANCE}/pyenv/bin/easy_install pip -U "pip>=1.0" "pip<=1.0.99"
+ echo "Attempting to install 'pip' 1.0 from pypi.python.org into pyenv to be used for extensions ..."
+ sudo -u ckan${INSTANCE} /var/lib/ckan/${INSTANCE}/pyenv/bin/easy_install --upgrade "pip>=1.0" "pip<=1.0.99"
+ echo "done."
+ cat <<- EOF > /var/lib/ckan/${INSTANCE}/packaging_version.txt
+ 1.5
+ EOF
cat <<- EOF > /var/lib/ckan/${INSTANCE}/wsgi.py
import os
instance_dir = '/var/lib/ckan/${INSTANCE}'
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 requires/lucid_conflict.txt
--- a/requires/lucid_conflict.txt
+++ b/requires/lucid_conflict.txt
@@ -9,7 +9,3 @@
sqlalchemy==0.6.6
webhelpers==1.2
-# NOTE: Developers, our build script for the Debian packages relies on the
-# requirements above being specified as editable resources with their
-# mercurial, SVN or git repository specified at the correct revision to
-# package.
diff -r e9d0ce90898e2b271a5fb2f31b4d516ea2742116 -r b3b54069ae2579c68a66e44a8a211599cd107a72 requires/lucid_missing.txt
--- a/requires/lucid_missing.txt
+++ b/requires/lucid_missing.txt
@@ -1,26 +1,21 @@
# These are packages that we rely on that aren't present in Lucid. We package
# them and put them in our own CKAN repository
+# Packages we install from source (could perhaps use release versions)
# pyutilib.component.core>=4.1,<4.1.99
-e svn+https://software.sandia.gov/svn/public/pyutilib/pyutilib.component.core/trunk@1972#egg=pyutilib.component.core
-# licenses==0.4,<0.6.99
--e hg+https://bitbucket.org/okfn/licenses@0eed4a13296b#egg=licenses
# vdm>=0.9,<0.9.99
-e hg+https://bitbucket.org/okfn/vdm@vdm-0.9#egg=vdm
-# markupsafe==0.9.2 required by webhelpers==1.2 required by formalchemy with SQLAlchemy 0.6
-markupsafe==0.9.2
# autoneg>=0.5
-e git+https://github.com/wwaites/autoneg.git@b4c727b164f411cc9d60#egg=autoneg
# flup>=0.5
-e hg+http://hg.saddi.com/flup@301a58656bfb#egg=flup
-# solrpy == 0.9.4
+
+# Packages already on pypi.python.org
solrpy==0.9.4
-# FormAlchemy
formalchemy==1.3.9
-# Apachemiddleware
--e hg+https://hg.3aims.com/public/ApacheMiddleware@tip#egg=apachemiddleware
+apachemiddleware==0.1.1
+licenses==0.6.1
+# markupsafe is required by webhelpers==1.2 required by formalchemy with SQLAlchemy 0.6
+markupsafe==0.9.2
-# NOTE: Developers, our build script for the Debian packages relies on the
-# requirements above being specified as editable resources with their
-# mercurial, SVN or git repository specified at the correct revision to
-# package
https://bitbucket.org/okfn/ckan/changeset/1d4065644baa/
changeset: 1d4065644baa
branch: release-v1.5
user: thejimmyg
date: 2011-11-04 16:58:09
summary: [merge]
affected #: 2 files
diff -r b3b54069ae2579c68a66e44a8a211599cd107a72 -r 1d4065644baaa3e8cb9ab9e8e54a62eed6fb449f ckan/logic/auth/create.py
--- a/ckan/logic/auth/create.py
+++ b/ckan/logic/auth/create.py
@@ -8,6 +8,7 @@
user = context['user']
check1 = check_access_old(model.System(), model.Action.PACKAGE_CREATE, context)
+
if not check1:
return {'success': False, 'msg': _('User %s not authorized to create packages') % str(user)}
else:
@@ -84,10 +85,14 @@
if context.get("allow_partial_update"):
return True
- group_dicts = data_dict.get("groups", [])
+ group_blobs = data_dict.get("groups", [])
groups = set()
- for group_dict in group_dicts:
- id = group_dict.get('id')
+ for group_blob in group_blobs:
+ # group_blob might be a dict or a group_ref
+ if isinstance(group_blob, dict):
+ id = group_blob.get('id')
+ else:
+ id = group_blob
if not id:
continue
grp = model.Group.get(id)
diff -r b3b54069ae2579c68a66e44a8a211599cd107a72 -r 1d4065644baaa3e8cb9ab9e8e54a62eed6fb449f ckan/tests/functional/api/model/test_package.py
--- a/ckan/tests/functional/api/model/test_package.py
+++ b/ckan/tests/functional/api/model/test_package.py
@@ -230,6 +230,25 @@
assert_equal(data['name'], data_returned['name'])
assert_equal(data['license_id'], data_returned['license_id'])
+ def test_entity_get_then_post_new(self):
+ offset = self.package_offset(self.war.name)
+ res = self.app.get(offset, status=self.STATUS_200_OK)
+ data = self.loads(res.body)
+
+ # change name and create a new package
+ data['name'] = u'newpkg'
+ data['id'] = None # ensure this doesn't clash or you get 409 error
+ postparams = '%s=1' % self.dumps(data)
+ # use russianfan now because he has rights to add this package to
+ # the 'david' group.
+ extra_environ = {'REMOTE_USER': 'russianfan'}
+ res = self.app.post(self.package_offset(), params=postparams,
+ status=self.STATUS_201_CREATED,
+ extra_environ=extra_environ)
+ data_returned = self.loads(res.body)
+ assert_equal(data['name'], data_returned['name'])
+ assert_equal(data['license_id'], data_returned['license_id'])
+
def test_entity_post_changed_readonly(self):
# (ticket 662) Edit a readonly field gives error
offset = self.package_offset(self.war.name)
Repository URL: https://bitbucket.org/okfn/ckan/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list