[ckan-changes] commit/ckanext-dgu: 5 new changesets
Bitbucket
commits-noreply at bitbucket.org
Fri Jul 15 17:09:05 UTC 2011
5 new changesets in ckanext-dgu:
http://bitbucket.org/okfn/ckanext-dgu/changeset/bd13f68d873e/
changeset: bd13f68d873e
user: dread
date: 2011-06-29 17:07:58
summary: [testtools]: #959 Requesting of parent department for each organisation.
affected #: 8 files (5.3 KB)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckanext/dgu/bin/xmlrpc_command.py Wed Jun 29 17:07:58 2011 +0200
@@ -0,0 +1,35 @@
+from ckanext.dgu.bin.command import Command
+
+class XmlRpcCommand(Command):
+ '''Derive from this to use the xmlrpc setting options.
+ '''
+ def __init__(self, usage=None):
+ super(XmlRpcCommand, self).__init__()
+
+ def add_options(self):
+ super(XmlRpcCommand, self).add_options()
+ self.parser.add_option("-X", "--xmlrpc-url",
+ dest="xmlrpc_url",
+ )
+ self.parser.add_option("-D", "--xmlrpc-domain",
+ dest="xmlrpc_domain",
+ )
+ self.parser.add_option("-U", "--xmlrpc-username",
+ dest="xmlrpc_username",
+ )
+ self.parser.add_option("-P", "--xmlrpc-password",
+ dest="xmlrpc_password",
+ )
+
+ def command(self):
+ if not (self.options.xmlrpc_domain or self.options.xmlrpc_url):
+ self.parser.error('Please specify an XML RPC domain or URL')
+
+ self.xmlrpc_settings = {
+ 'xmlrpc_url':self.options.xmlrpc_url,
+ 'xmlrpc_domain':self.options.xmlrpc_domain,
+ 'xmlrpc_username':self.options.xmlrpc_username,
+ 'xmlrpc_password':self.options.xmlrpc_password}
+
+ # now do command
+
--- a/ckanext/dgu/command.py Wed Jun 22 17:14:36 2011 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,36 +0,0 @@
-from ckanext.importlib.command import Command
-
-class XmlRpcCommand(Command):
- '''Derive from this to use the xmlrpc setting options.
- '''
- def __init__(self, usage=None):
- self.add_options()
- super(XmlRpcCommand, self).__init__()
-
- def add_options(self):
- super(XmlRpcCommand, self).add_options()
- self.parser.add_option("-X", "--xmlrpc-url",
- dest="xmlrpc_url",
- )
- self.parser.add_option("-D", "--xmlrpc-domain",
- dest="xmlrpc_domain",
- )
- self.parser.add_option("-U", "--xmlrpc-username",
- dest="xmlrpc_username",
- )
- self.parser.add_option("-P", "--xmlrpc-password",
- dest="xmlrpc_password",
- )
-
- def command(self):
- if not (self.options.xmlrpc_domain or self.options.xmlrpc_url):
- self.parser.error('Please specify an XML RPC domain or URL')
-
- self.xmlrpc_settings = {
- 'xmlrpc_url':self.options.xmlrpc_url,
- 'xmlrpc_domain':self.options.xmlrpc_domain,
- 'xmlrpc_username':self.options.xmlrpc_username,
- 'xmlrpc_password':self.options.xmlrpc_password}
-
- # now do command
-
--- a/ckanext/dgu/drupalclient.py Wed Jun 22 17:14:36 2011 +0100
+++ b/ckanext/dgu/drupalclient.py Wed Jun 29 17:07:58 2011 +0200
@@ -77,17 +77,27 @@
log.info('Obtained Drupal session for session ID %r: %r', session_id, session)
return session
- def get_department_from_publisher(self, id):
+ def get_department_from_organisation(self, id):
try:
- department = self.drupal.organisation.department(str(id))
+ # Example response:
+ # {'11419': 'Department for Culture, Media and Sport'}
+ dept_dict = self.drupal.organisation.department(str(id))
except socket.error, e:
raise DrupalRequestError('Socket error with url \'%s\': %r' % (self.xmlrpc_url, e))
except Fault, e:
- raise DrupalRequestError('Drupal returned error for organisation_id %r: %r' % (id, e))
+ if e.faultCode == 404:
+ raise DrupalKeyError(id)
+ else:
+ raise DrupalRequestError('Drupal returned error for organisation_id %r: %r' % (id, e))
except ProtocolError, e:
raise DrupalRequestError('Drupal returned protocol error for organisation_id %r: %r' % (id, e))
- log.info('Obtained Drupal parent department %r from publisher %r', department, id)
- return department
+ if len(dept_dict) == 0:
+ raise DrupalKeyError('No parent department for organisation_id %r' % id)
+ if len(dept_dict) > 1:
+ raise DrupalKeyError('Multiple parent departments for organisation_id %r: %r' % (id, dept_dict))
+ department_id, department_name = dept_dict.items()[0]
+ log.info('Obtained Drupal parent department %r (%s) from organisation %r', department_name, department_id, id)
+ return department_id
def get_organisation_name(self, id):
try:
--- a/ckanext/dgu/tests/test_drupalclient.py Wed Jun 22 17:14:36 2011 +0100
+++ b/ckanext/dgu/tests/test_drupalclient.py Wed Jun 29 17:07:58 2011 +0200
@@ -44,3 +44,17 @@
assert_raises(DrupalKeyError, client.get_organisation_name, '999')
assert_raises(DrupalKeyError, client.get_organisation_name, '')
assert_raises(DrupalKeyError, client.get_organisation_name, None)
+
+ def test_get_department_from_organisation(self):
+ drupal_config = get_mock_drupal_config()
+ client = DrupalClient()
+
+ parent_org_id = client.get_department_from_organisation('2')
+ assert_equal(parent_org_id, '7')
+ parent_org_id = client.get_department_from_organisation(2)
+ assert_equal(parent_org_id, '7')
+
+ assert_raises(DrupalKeyError, client.get_department_from_organisation, '999')
+ assert_raises(DrupalKeyError, client.get_department_from_organisation, '')
+ assert_raises(DrupalKeyError, client.get_department_from_organisation, None)
+
--- a/ckanext/dgu/tests/testtools/test_mock_drupal.py Wed Jun 22 17:14:36 2011 +0100
+++ b/ckanext/dgu/tests/testtools/test_mock_drupal.py Wed Jun 29 17:07:58 2011 +0200
@@ -13,7 +13,7 @@
assert isinstance(user, dict)
assert user.has_key('name')
assert user.has_key('publishers')
- assert isinstance(user['publishers'], dict)
+ assert isinstance(user['publishers'], dict), user['publishers']
publisher = user['publishers'].items()[0]
assert isinstance(publisher[0], str)
assert_equal(publisher[0], '1')
@@ -39,9 +39,9 @@
assert_raises(xmlrpclib.Fault, self.s.organisation.match, 'Wrong org name')
def test_department_lookup(self):
- # return org id by name
+ # return org id of the parent department
org = self.s.organisation.department('2')
- assert_equal(org, {'1': 'National Health Service'})
+ assert_equal(org, {'7': 'Department of Health'})
def test_department_lookup_error(self):
assert_raises(xmlrpclib.Fault, self.s.organisation.department, '9999')
--- a/ckanext/dgu/testtools/lots_of_orgs.json Wed Jun 22 17:14:36 2011 +0100
+++ b/ckanext/dgu/testtools/lots_of_orgs.json Wed Jun 29 17:07:58 2011 +0200
@@ -1,1 +1,1 @@
-{"16476": "Department of Finance and Personnel", "16337": "Office of the First and Deputy First Minister", "16479": "Department for Employment and Learning", "16478": "Department for Regional Development", "16203": "Higher Education Statistics Agency", "12076": "Health Protection Agency", "12077": "General Register Office for Scotland", "16450": "Department of the Environment", "11406": "Department for Transport", "11405": "Department of Energy and Climate Change", "11404": "Ministry of Defence", "11403": "Department for Environment, Food and Rural Affairs", "11402": "Home Office", "11401": "Department for Communities and Local Government", "11400": "Ministry of Justice", "11409": "Department for Work and Pensions", "16117": "Eurostat", "12158": "Office of Qualifications and Examinations Regulation", "11421": "Department for International Development", "11424": "Department for Education", "12367": "Food Standards Agency", "13015": "Health and Safety Executive", "12953": "Child Maintenance and Enforcement Commission", "16121": "NHS Information Centre for Health and Social Care", "16482": "Department for Social Development", "16329": "Northern Ireland Statistics and Research Agency", "16266": "National Treatment Agency for Substance Misuse", "16448": "NHS National Services Scotland", "12243": "Forestry Commission", "16480": "Department of Health, Social Services and Public Safety", "16220": "NHS Scotland", "16221": "ISD Scotland", "16348": "Police Service of Northern Ireland", "11410": "Department of Health", "11412": "Her Majesty's Revenue and Customs", "11413": "Welsh Assembly Government", "11414": "Scottish Government", "11415": "Northern Ireland Executive", "11418": "Her Majesty's Treasury", "11419": "Department for Culture, Media and Sport", "16449": "Department of Culture, Arts and Leisure", "11399": "Department for Business, Innovation and Skills", "16114": "Civil Aviation Authority", "16346": "Passenger Focus", "11606": "Office for National Statistics", "16113": "Department of Enterprise Trade and Investment"}
\ No newline at end of file
+{"16476": {"parent_department_id": "16476", "name": "Department of Finance and Personnel"}, "16337": {"parent_department_id": "11415", "name": "Office of the First and Deputy First Minister"}, "16479": {"parent_department_id": "16479", "name": "Department for Employment and Learning"}, "16478": {"parent_department_id": "16478", "name": "Department for Regional Development"}, "16203": {"parent_department_id": "11399", "name": "Higher Education Statistics Agency"}, "12076": {"parent_department_id": "11410", "name": "Health Protection Agency"}, "12077": {"parent_department_id": "12077", "name": "General Register Office for Scotland"}, "16450": {"parent_department_id": "16450", "name": "Department of the Environment"}, "11406": {"parent_department_id": "11406", "name": "Department for Transport"}, "11405": {"parent_department_id": "11405", "name": "Department of Energy and Climate Change"}, "11404": {"parent_department_id": "11404", "name": "Ministry of Defence"}, "11403": {"parent_department_id": "11403", "name": "Department for Environment, Food and Rural Affairs"}, "11402": {"parent_department_id": "11402", "name": "Home Office"}, "11401": {"parent_department_id": "11401", "name": "Department for Communities and Local Government"}, "11400": {"parent_department_id": "11400", "name": "Ministry of Justice"}, "11409": {"parent_department_id": "11409", "name": "Department for Work and Pensions"}, "16117": {"parent_department_id": "16115", "name": "Eurostat"}, "12158": {"parent_department_id": "11424", "name": "Office of Qualifications and Examinations Regulation"}, "11421": {"parent_department_id": "11421", "name": "Department for International Development"}, "11424": {"parent_department_id": "11424", "name": "Department for Education"}, "12367": {"parent_department_id": "12367", "name": "Food Standards Agency"}, "13015": {"parent_department_id": "11409", "name": "Health and Safety Executive"}, "12953": {"parent_department_id": "11409", "name": "Child Maintenance and Enforcement Commission"}, "16121": {"parent_department_id": "11410", "name": "NHS Information Centre for Health and Social Care"}, "16482": {"parent_department_id": "16482", "name": "Department for Social Development"}, "16329": {"parent_department_id": "11415", "name": "Northern Ireland Statistics and Research Agency"}, "16266": {"parent_department_id": "11410", "name": "National Treatment Agency for Substance Misuse"}, "16448": {"parent_department_id": "11414", "name": "NHS National Services Scotland"}, "12243": {"parent_department_id": "11403", "name": "Forestry Commission"}, "16480": {"parent_department_id": "16480", "name": "Department of Health, Social Services and Public Safety"}, "16220": {"parent_department_id": "11410", "name": "NHS Scotland"}, "16221": {"parent_department_id": "11410", "name": "ISD Scotland"}, "16348": {"parent_department_id": "11415", "name": "Police Service of Northern Ireland"}, "11410": {"parent_department_id": "11410", "name": "Department of Health"}, "11412": {"parent_department_id": "11412", "name": "Her Majesty's Revenue and Customs"}, "11413": {"parent_department_id": "11413", "name": "Welsh Assembly Government"}, "11414": {"parent_department_id": "11414", "name": "Scottish Government"}, "11415": {"parent_department_id": "11415", "name": "Northern Ireland Executive"}, "11418": {"parent_department_id": "11418", "name": "Her Majesty's Treasury"}, "11419": {"parent_department_id": "11419", "name": "Department for Culture, Media and Sport"}, "16449": {"parent_department_id": "16449", "name": "Department of Culture, Arts and Leisure"}, "11399": {"parent_department_id": "11399", "name": "Department for Business, Innovation and Skills"}, "16114": {"parent_department_id": "11406", "name": "Civil Aviation Authority"}, "16346": {"parent_department_id": "11406", "name": "Passenger Focus"}, "11606": {"parent_department_id": "11408", "name": "Office for National Statistics"}, "16113": {"parent_department_id": "11415", "name": "Department of Enterprise Trade and Investment"}}
\ No newline at end of file
--- a/ckanext/dgu/testtools/mock_drupal.py Wed Jun 22 17:14:36 2011 +0100
+++ b/ckanext/dgu/testtools/mock_drupal.py Wed Jun 29 17:07:58 2011 +0200
@@ -4,7 +4,7 @@
import paste.script
from ckanext.dgu.testtools.organisations import test_organisations, \
- LotsOfOrganisations
+ test_organisation_names, LotsOfOrganisations
# NB Mock drupal details must match those in ckanext-dgu/test-core.ini
MOCK_DRUPAL_PATH = '/services/xmlrpc'
@@ -19,7 +19,7 @@
'rpc_port': MOCK_DRUPAL_PORT,
'test_users': {'62': {'name': 'testname',
'uid': '62',
- 'publishers': test_organisations}
+ 'publishers': test_organisation_names}
},
'test_sessions': {'4160a72a4d6831abec1ac57d7b5a59eb': '62'}
}
@@ -128,7 +128,7 @@
# Example response:
# "Arts Council England"
try:
- return self.organisations[org_id]
+ return self.organisations[org_id]['name']
except KeyError:
raise Fault(404, 'There is no organisation with such ID.')
@@ -137,20 +137,19 @@
# return org id by name
# Example response:
# "12022"
- for id, name in self.organisations.items():
- if name == org_name:
+ for id, org_dict in self.organisations.items():
+ if org_name == org_dict['name']:
return id
raise Fault(404, 'Cannot find organisation %r.' % org_name)
@classmethod
def department(cls, org_id):
- # return top level parent ord id by org id
+ # return top level parent org id by org id
# Example response:
# {'11419': 'Department for Culture, Media and Sport'}
- if org_id == '2':
- return {'1': 'National Health Service'}
- elif org_id in self.organisations:
- return {org_id: self.organisations[org_id]}
+ if org_id in self.organisations:
+ parent_org_id = self.organisations[org_id]['parent_department_id']
+ return {parent_org_id: self.organisations[parent_org_id]['name']}
else:
raise Fault(404, 'No department for organisation ID %r' % org_id)
--- a/ckanext/dgu/testtools/organisations.py Wed Jun 22 17:14:36 2011 +0100
+++ b/ckanext/dgu/testtools/organisations.py Wed Jun 29 17:07:58 2011 +0200
@@ -2,7 +2,7 @@
import logging
import sys
-from ckanext.dgu.command import XmlRpcCommand
+from ckanext.dgu.bin.xmlrpc_command import XmlRpcCommand
from ckan.lib.helpers import json
@@ -12,14 +12,24 @@
log = logging.getLogger(__name__)
-test_organisations = {'1': 'National Health Service',
- '2': 'Ealing PCT',
- '3': 'Department for Education',
- '4': 'Department of Energy and Climate Change',
- '5': 'Department for Business, Innovation and Skills',
- '6': 'Department for Communities and Local Government',
+test_organisations = {'1': {'name': 'National Health Service',
+ 'parent_department_id': '7'},
+ '2': {'name': 'Ealing PCT',
+ 'parent_department_id': '7'},
+ '3': {'name': 'Department for Education',
+ 'parent_department_id': '3'},
+ '4': {'name': 'Department of Energy and Climate Change',
+ 'parent_department_id': '4'},
+ '5': {'name': 'Department for Business, Innovation and Skills',
+ 'parent_department_id': '5'},
+ '6': {'name': 'Department for Communities and Local Government',
+ 'parent_department_id': '6'},
+ '7': {'name': 'Department of Health',
+ 'parent_department_id': '7'},
}
+test_organisation_names = dict([(id, org_dict['name']) for id, org_dict in test_organisations.items()])
+
class LotsOfOrganisations(object):
orgs_cache = {}
lots_of_orgs_filepath = os.path.join(os.path.dirname(__file__),
@@ -48,7 +58,9 @@
has_errors = True
continue
proper_org_name = drupal.get_organisation_name(org_id)
- orgs[org_id] = proper_org_name
+ parent_department_id = drupal.get_department_from_organisation(org_id)
+ orgs[org_id] = {'name': proper_org_name,
+ 'parent_department_id': parent_department_id}
f = open(cls.lots_of_orgs_filepath, 'w')
try:
@@ -71,7 +83,7 @@
def command(self):
super(OrgCommand, self).command()
- cmd = LotsOfOrganisations.generate(xmlrpc_settings)
+ cmd = LotsOfOrganisations.generate(self.xmlrpc_settings)
def command():
OrgCommand().command()
http://bitbucket.org/okfn/ckanext-dgu/changeset/1a9425b60c8c/
changeset: 1a9425b60c8c
user: dread
date: 2011-07-06 20:25:17
summary: [forms]: Changed where we defined content types in ckan.
affected #: 1 file (19 bytes)
--- a/ckanext/dgu/forms/formapi.py Wed Jun 29 17:07:58 2011 +0200
+++ b/ckanext/dgu/forms/formapi.py Wed Jul 06 19:25:17 2011 +0100
@@ -9,7 +9,7 @@
import ckan.controllers.package
from ckan.lib.package_saver import WritePackageFromBoundFieldset
from ckan.lib.package_saver import ValidationException
-from ckan.controllers.api import ApiController
+from ckan.controllers.api import ApiController, CONTENT_TYPES
from ckanext.dgu.forms import harvest_source as harvest_source_form
from ckanext.dgu.drupalclient import DrupalClient, DrupalXmlRpcSetupError, \
@@ -448,7 +448,7 @@
if err_msg:
self.log.debug(err_msg)
response.status_int = 400
- response.headers['Content-Type'] = self.content_type_json
+ response.headers['Content-Type'] = CONTENT_TYPES['json']
return json.dumps(err_msg)
else:
return self._finish_ok(job)
@@ -478,7 +478,7 @@
if err_msg:
self.log.debug(err_msg)
response.status_int = 400
- response.headers['Content-Type'] = self.content_type_json
+ response.headers['Content-Type'] = CONTENT_TYPES['json']
return json.dumps(err_msg)
else:
return self._finish_ok()
@@ -567,9 +567,9 @@
log.error("Couldn't update harvest source: %s" % traceback.format_exc())
raise
- def get_department_from_publisher(self, id):
+ def get_department_from_organisation(self, id):
try:
- department = self.__class__._drupal_client().get_department_from_publisher(id)
+ department = self.__class__._drupal_client().get_department_from_organisation(id)
except DrupalRequestError, e:
abort(500, 'Error making internal request: %r' % e)
return self._finish_ok(department)
http://bitbucket.org/okfn/ckanext-dgu/changeset/8648ce617b87/
changeset: 8648ce617b87
user: dread
date: 2011-07-15 19:01:32
summary: [tests]: Resource stream - moved and provided tests. Tests are currently only for harvest filter (was going to be for archive link too, but that was scrapped).
affected #: 6 files (8.4 KB)
--- a/ckanext/dgu/plugin.py Wed Jul 06 19:25:17 2011 +0100
+++ b/ckanext/dgu/plugin.py Fri Jul 15 18:01:32 2011 +0100
@@ -10,12 +10,7 @@
from ckanext.dgu.middleware import AuthAPIMiddleware
import ckanext.dgu
-from ckan.model import Session
-from ckanext.harvest.model import HarvestObject
-
-import ckanext.dgu.forms.html as html
-from genshi.input import HTML
-from genshi.filters import Transformer
+import stream_filters
log = getLogger(__name__)
@@ -99,7 +94,7 @@
map.connect('/api/rest/harvesteddocument/:id/html', controller='ckanext.dgu.forms.formapi:FormController',
action='harvested_document_view_format', format='html')
"""
- map.connect('/api/2/util/publisher/:id/department', controller='ckanext.dgu.forms.formapi:FormController', action='get_department_from_publisher')
+ map.connect('/api/2/util/publisher/:id/department', controller='ckanext.dgu.forms.formapi:FormController', action='get_department_from_organisation')
map.connect('/', controller='ckanext.dgu.controllers.catalogue:CatalogueController', action='home')
map.connect('home', '/ckan/', controller='home', action='index')
return map
@@ -119,25 +114,14 @@
from pylons import request, tmpl_context as c
routes = request.environ.get('pylons.routes_dict')
- if routes.get('controller') == 'package' and \
- routes.get('action') == 'read' and c.pkg.id:
+ if routes and \
+ routes.get('controller') == 'package' and \
+ routes.get('action') == 'read' and c.pkg.id:
is_inspire = [v[1] for i,v in enumerate(c.pkg_extras) if v[0] == 'INSPIRE']
if is_inspire and is_inspire[0] == 'True':
- # We need the guid from the HarvestedObject!
- doc = Session.query(HarvestObject). \
- filter(HarvestObject.package_id==c.pkg.id). \
- order_by(HarvestObject.metadata_modified_date.desc()). \
- order_by(HarvestObject.gathered.desc()). \
- limit(1).first()
- if doc:
- data = {'guid': doc.guid}
- html_code = html.GEMINI_CODE
- if len(c.pkg.resources) == 0:
- # If no resources, the table has only two columns
- html_code = html_code.replace('<td></td>','')
+ stream = stream_filters.harvest_filter(stream, c.pkg)
- stream = stream | Transformer('body//div[@class="resources subsection"]/table')\
- .append(HTML(html_code % data))
+ stream = stream_filters.archive_filter(stream)
return stream
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckanext/dgu/stream_filters.py Fri Jul 15 18:01:32 2011 +0100
@@ -0,0 +1,27 @@
+import re
+
+from genshi.filters.transform import Transformer
+from genshi.input import HTML
+
+from ckan.model import Session
+from ckanext.harvest.model import HarvestObject
+import ckanext.dgu.forms.html as html
+
+def harvest_filter(stream, pkg):
+ # We need the guid from the HarvestedObject!
+ doc = Session.query(HarvestObject). \
+ filter(HarvestObject.package_id==pkg.id). \
+ order_by(HarvestObject.metadata_modified_date.desc()). \
+ order_by(HarvestObject.gathered.desc()). \
+ limit(1).first()
+ if doc:
+ data = {'guid': doc.guid}
+ html_code = html.GEMINI_CODE
+ if len(pkg.resources) == 0:
+ # If no resources, the table has only two columns
+ html_code = html_code.replace('<td></td>','')
+
+ stream = stream | Transformer('body//div[@class="resources subsection"]/table')\
+ .append(HTML(html_code % data))
+ return stream
+
--- a/ckanext/dgu/tests/__init__.py Wed Jul 06 19:25:17 2011 +0100
+++ b/ckanext/dgu/tests/__init__.py Fri Jul 15 18:01:32 2011 +0100
@@ -4,27 +4,17 @@
from paste.script.appinstall import SetupCommand
from pylons import config
+from ckan import model
from ckan.lib.create_test_data import CreateTestData
-from ckan.tests import WsgiAppCase
+from ckan.tests import WsgiAppCase, BaseCase
from ckanext.dgu.testtools.mock_drupal import MOCK_DRUPAL_URL
+from ckanext.harvest.model import HarvestJob, HarvestObject
+from ckanext.harvest.model import setup as harvest_setup
# Invoke websetup with the current config file
SetupCommand('setup-app').run([config['__file__']])
-class BaseCase(object):
- @staticmethod
- def _system(cmd):
- import commands
- (status, output) = commands.getstatusoutput(cmd)
- if status:
- raise Exception, "Couldn't execute cmd: %s: %s" % (cmd, output)
-
- @classmethod
- def _paster(cls, cmd, config_path_rel):
- cls._system('paster --plugin ckanext-dgu %s' % cmd)
-
-
class PackageFixturesBase:
def create(self, **kwargs):
CreateTestData.create_arbitrary(self.pkgs,
@@ -190,6 +180,8 @@
extra_keys = set(dict_to_check.keys()) - set(expected_dict.keys())
assert not extra_keys, 'Keys that should not be there: %r. All unmatching keys: %r' % (extra_keys, unmatching_keys)
+class DrupalSetupError(Exception):
+ pass
class MockDrupalCase(BaseCase):
xmlrpc_url = MOCK_DRUPAL_URL
@@ -197,6 +189,7 @@
@classmethod
def setup_class(cls):
+ cls._check_drupal_not_already_running()
cls.process = cls._mock_drupal_start()
cls._wait_for_drupal_to_start()
@@ -214,6 +207,21 @@
return process
@classmethod
+ def _check_drupal_not_already_running(cls,
+ url=None):
+ import xmlrpclib
+ import socket
+ import time
+
+ url = url or cls.xmlrpc_url
+ drupal = xmlrpclib.ServerProxy(url)
+ try:
+ response = drupal.system.listMethods()
+ except socket.error, e:
+ return
+ raise DrupalSetupError('Drupal already seems to be running: %s' % url)
+
+ @classmethod
def _wait_for_drupal_to_start(cls,
url=None,
timeout=15):
@@ -229,7 +237,8 @@
except socket.error, e:
time.sleep(0.01)
else:
- break
+ return
+ raise DrupalSetupError('Time-out while waiting for Drupal to start up.')
@classmethod
def _mock_drupal_stop(cls, process):
@@ -241,3 +250,25 @@
def strip_organisation_id(org_name_with_id):
# e.g. 'NHS [54]' becomes 'NHS [some_number]'
return re.sub('\[\d+\]', '[some_number]', org_name_with_id)
+
+class HarvestFixture(object):
+ '''Base class, useful for several tests relating to harvesting.'''
+ @classmethod
+ def setup_class(cls):
+ # Create package and its harvest object
+ CreateTestData.create()
+ harvest_setup()
+ job = HarvestJob()
+ job.save()
+ model.repo.commit_and_remove()
+ job = model.Session.query(HarvestJob).first()
+ ho = HarvestObject(package=model.Package.by_name(u'annakarenina'),
+ harvest_job=job,
+ guid='test-guid',
+ content='<xml>test content</xml>')
+ ho.save()
+ model.repo.commit_and_remove()
+
+ @classmethod
+ def teardown_class(cls):
+ model.repo.rebuild_db()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckanext/dgu/tests/functional/test_filters.py Fri Jul 15 18:01:32 2011 +0100
@@ -0,0 +1,74 @@
+from genshi.input import HTML
+from nose.tools import assert_equal
+
+from ckan.lib.package_saver import PackageSaver
+from ckan.lib.create_test_data import CreateTestData
+from ckan.lib.base import render, c, h, g
+import ckan.logic.action.get as get
+from ckan import model
+
+from ckanext.dgu.stream_filters import harvest_filter
+
+from ckan.tests import TestController as ControllerTestCase
+from ckan.tests.pylons_controller import PylonsTestCase
+from ckan.tests.html_check import HtmlCheckMethods
+from ckanext.dgu.tests import HarvestFixture
+
+class TestHarvestFilter(PylonsTestCase, HtmlCheckMethods,
+ ControllerTestCase, HarvestFixture):
+ @classmethod
+ def setup_class(cls):
+ PylonsTestCase.setup_class()
+ HarvestFixture.setup_class()
+
+ # prepare to render package page
+ user = model.PSEUDO_USER__VISITOR
+ c.pkg = model.Package.by_name(u'annakarenina')
+ c.locale = 'en'
+ c.body_class = 'hide-sidebar'
+ c.controller = 'package'
+ c.action = 'read'
+ c.user = user
+ c.is_preview = False
+ c.hide_welcome_message = False
+ context = {'model': model, 'session': model.Session,
+ 'user': c.user,
+ 'id':c.pkg.id,
+ 'package':c.pkg}
+ c.pkg_dict = get.package_show(context)
+
+ # Render package view page
+ # (filter should not be called on this occasion)
+ PackageSaver().render_package(c.pkg_dict,
+ context)
+ cls.pkg_page = render('package/read.html')
+
+ # Expected URLs
+ cls.harvest_xml_url = '/api/2/rest/harvestobject/test-guid/xml'
+ cls.harvest_html_url = '/api/2/rest/harvestobject/test-guid/html'
+
+ def test_basic(self):
+ pkg_id = c.pkg.id
+
+ # before filter
+ # <a href="http://www.annakarenina.com/download/x=1&y=2" target="_blank">Full text. Needs escaping: " Umlaut: u</a>
+ self.check_named_element(self.pkg_page, 'a', '!href="%s"' % self.harvest_xml_url)
+ self.check_named_element(self.pkg_page, 'a', '!href="%s"' % self.harvest_html_url)
+
+ res = harvest_filter(HTML(self.pkg_page), c.pkg)
+ res = res.render('html').decode('utf8')
+ main_res = self.main_div(res)
+
+ # after filter
+ self.check_named_element(res, 'a', 'href="%s"' % self.harvest_xml_url)
+ self.check_named_element(res, 'a', 'href="%s"' % self.harvest_html_url)
+
+ def test_link_to_xml(self):
+ res = self.app.get(self.harvest_xml_url)
+ assert_equal(res.body, '<xml>test content</xml>')
+
+ def test_link_to_html(self):
+ res = self.app.get(self.harvest_html_url)
+ assert 'GEMINI record' in res.body
+ assert 'error' not in res.body
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckanext/dgu/tests/test_filters.py Fri Jul 15 18:01:32 2011 +0100
@@ -0,0 +1,82 @@
+from genshi.input import HTML
+
+from ckan import model
+from ckan.lib.create_test_data import CreateTestData
+from ckanext.dgu.stream_filters import harvest_filter
+
+from ckanext.dgu.tests import HarvestFixture
+from ckan.tests.html_check import HtmlCheckMethods
+
+basic_package_page = '''
+<html>
+ <body class="hide-sidebar">
+ <!-- Downloads and resources -->
+ <div class="resources subsection">
+ <h3>Downloads & Resources</h3>
+ <table>
+ <tr>
+ <th>Description</th>
+ <th>Format</th>
+ <th>Hash</th>
+ </tr>
+ <tr>
+ <td>
+ <a href="http://site.com/data.csv" target="_blank">Description of the data</a>
+ </td>
+ <td>csv</td>
+ <td>abc123</td>
+ </tr>
+ </table>
+ </div>
+ </body>
+<html>
+'''
+
+class TestHarvestFilter(HtmlCheckMethods, HarvestFixture):
+ @classmethod
+ def setup_class(cls):
+ HarvestFixture.setup_class()
+ cls.pkg_page = HTML(basic_package_page)
+
+ def test_basic(self):
+ harvest_xml_url = '/api/2/rest/harvestobject/test-guid/xml'
+ harvest_html_url = '/api/2/rest/harvestobject/test-guid/html'
+
+ # before filter
+ pkg_page = HTML(self.pkg_page).render()
+ self.check_named_element(pkg_page, 'a', '!href="%s"' % harvest_xml_url)
+ self.check_named_element(pkg_page, 'a', '!href="%s"' % harvest_html_url)
+
+ anna = model.Package.by_name(u'annakarenina')
+ res = harvest_filter(HTML(self.pkg_page), anna)
+ res = res.render('html').decode('utf8')
+ print res
+
+ # after filter
+ self.check_named_element(res, 'a', 'href="%s"' % harvest_xml_url)
+ self.check_named_element(res, 'a', 'href="%s"' % harvest_html_url)
+
+
+# test disabled - archive filter never written
+class _TestArchiveFilter(HtmlCheckMethods):
+ @classmethod
+ def setup_class(cls):
+ cls.pkg_page = HTML(basic_package_page)
+
+ def test_basic(self):
+ self.pkg_url = 'http://site.com/data.csv'
+ self.archive_url = 'http://webarchive.nationalarchives.gov.uk/tna/+/' + self.pkg_url
+
+ # before filter
+ # <a href="http://www.annakarenina.com/download/x=1&y=2" target="_blank">Full text. Needs escaping: " Umlaut: u</a>
+ self.pkg_page = HTML(self.pkg_page).render()
+ self.check_named_element(self.pkg_page, 'a', 'href="%s"' % self.pkg_url)
+ self.check_named_element(self.pkg_page, 'a', '!href="%s"' % self.archive_url)
+
+ res = archive_filter(HTML(self.pkg_page))
+ res = res.render('html').decode('utf8')
+ print res
+ # after filter
+ self.check_named_element(res, 'a', 'href="%s"' % self.pkg_url)
+ self.check_named_element(res, 'a', 'href="%s"' % self.archive_url)
+
--- a/test-core.ini Wed Jul 06 19:25:17 2011 +0100
+++ b/test-core.ini Fri Jul 15 18:01:32 2011 +0100
@@ -22,7 +22,7 @@
dgu.xmlrpc_username =
dgu.xmlrpc_password =
dgu.xmlrpc_domain = localhost:8051
-ckan.plugins = dgu_form_api form_api_tester
+ckan.plugins = dgu_form_api form_api_tester harvest inspire_api
# Logging configuration
http://bitbucket.org/okfn/ckanext-dgu/changeset/74a280313575/
changeset: 74a280313575
user: dread
date: 2011-07-15 19:03:56
summary: [publishers]: Preliminary work for publisher syncing from drupal.
affected #: 8 files (5.4 KB)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckanext/dgu/bin/sync_organisations.py Fri Jul 15 18:03:56 2011 +0100
@@ -0,0 +1,16 @@
+from ckanext.dgu.publishers import sync
+from ckanext.dgu.bin.xmlrpc_command import XmlRpcCommand
+
+class OrgCommand(XmlRpcCommand):
+ '''Syncs organisations from Drupal into CKAN groups.
+ '''
+ summary = 'Syncs organisations from Drupal into CKAN groups.'
+
+ def command(self):
+ super(OrgCommand, self).command()
+
+ sync.sync(self.xmlrpc_settings)
+
+def command():
+ OrgCommand().command()
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckanext/dgu/publishers/sync.py Fri Jul 15 18:03:56 2011 +0100
@@ -0,0 +1,82 @@
+import logging
+
+from ckan import model
+from ckan.lib.munge import munge_title_to_name
+from ckanext.dgu.drupalclient import DrupalClient, DrupalKeyError
+
+log = logging.getLogger(__name__)
+
+
+def munge_group_name(group_title):
+ return munge_title_to_name(group_title)
+
+def group_by_drupal_id(drupal_org_id):
+ return model.Session.query(model.Group) \
+ .join('_extras', aliased=True) \
+ .filter(model.GroupExtra.key=='drupal_id') \
+ .filter(model.GroupExtra.value==str(drupal_org_id)) \
+ .first()
+
+def start_revision(org_id):
+ rev = model.repo.new_revision()
+ rev.author = 'okfn_maintenance'
+ rev.message = 'Syncing organisation %s.' % org_id
+
+def end_revision():
+ if len(model.Session.dirty) > 1 or model.Session.new:
+ # changes have occurred (aside from the revision)
+ model.repo.commit_and_remove()
+ else:
+ # changes have not occurred, so delete revision
+ model.Session.rollback()
+
+
+def sync(xmlrpc_settings):
+ drupal_client = DrupalClient(xmlrpc_settings)
+
+ for org_id in range(10000, 20000):
+ sync_organisation(drupal_client, org_id)
+
+def sync_organisation(drupal_client, org_id):
+ '''Syncs a drupal organisation to a CKAN group. Will be recursive if
+ the parent organisation is not in CKAN yet.
+ Revisions and committing are handled by this function.
+ '''
+ org_id = str(org_id)
+ try:
+ org_name = drupal_client.get_organisation_name(org_id)
+ except DrupalKeyError:
+ log.warn('No organisation with id: %s', org_id)
+ return
+
+ parent_id = drupal_client.get_department_from_organisation(org_id)
+ group = group_by_drupal_id(org_id)
+ parent_group = group_by_drupal_id(parent_id)
+ if not parent_group and parent_id != org_id:
+ # if parent_id == org_id then we have to create the group
+ # object, but since the parent_group is the same object,
+ # we have to commit it before setting the department_id
+ # afterwards.
+ sync_organisation(drupal_client, parent_id)
+ parent_group = group_by_drupal_id(parent_id)
+ assert parent_group
+ start_revision(org_id)
+ if group:
+ group.name = munge_group_name(org_name)
+ group.title = org_name
+ group.extras['department_id'] = parent_group.id
+ group.extras['drupal_id'] = org_id
+ else:
+ group = model.Group(name=munge_group_name(org_name),
+ title=org_name)
+ if parent_group:
+ assert parent_id != org_id
+ group.extras['department_id'] = parent_group.id
+ group.extras['drupal_id'] = org_id
+ model.Session.add(group)
+ end_revision()
+ if not parent_group and parent_id == org_id:
+ start_revision(org_id)
+ group = group_by_drupal_id(org_id)
+ group.extras['department_id'] = group.id
+ end_revision()
--- a/ckanext/dgu/tests/forms/test_form_api.py Fri Jul 15 18:01:32 2011 +0100
+++ b/ckanext/dgu/tests/forms/test_form_api.py Fri Jul 15 18:03:56 2011 +0100
@@ -7,10 +7,7 @@
from ckan.tests import *
from ckan.tests import search_related
-from ckan.tests.functional.api.base import ApiTestCase
-from ckan.tests.functional.api.test_model import Api1TestCase
-from ckan.tests.functional.api.test_model import Api2TestCase
-from ckan.tests.functional.api.test_model import ApiUnversionedTestCase
+from ckan.tests.functional.api.base import ApiTestCase, Api1TestCase, Api2TestCase, ApiUnversionedTestCase
from ckanext.harvest.lib import get_harvest_source, create_harvest_source
--- a/ckanext/dgu/tests/forms/test_form_api_gov3.py Fri Jul 15 18:01:32 2011 +0100
+++ b/ckanext/dgu/tests/forms/test_form_api_gov3.py Fri Jul 15 18:03:56 2011 +0100
@@ -9,11 +9,9 @@
from ckan.lib.helpers import literal
import ckan.model as model
from ckan.lib.create_test_data import CreateTestData
-from test_form_api import BaseFormsApiCase, Api1TestCase, Api2TestCase
+from test_form_api import BaseFormsApiCase
from ckanext.dgu.tests import MockDrupalCase, Gov3Fixtures, strip_organisation_id
-from ckan.tests.functional.api.test_model import Api1TestCase
-from ckan.tests.functional.api.test_model import Api2TestCase
-from ckan.tests.functional.api.test_model import ApiUnversionedTestCase
+from ckan.tests.functional.api.base import ApiTestCase, Api1TestCase, Api2TestCase, ApiUnversionedTestCase
class PackageFixturesBase:
--- a/ckanext/dgu/tests/functional/test_package_gov3.py Fri Jul 15 18:01:32 2011 +0100
+++ b/ckanext/dgu/tests/functional/test_package_gov3.py Fri Jul 15 18:03:56 2011 +0100
@@ -93,8 +93,8 @@
class TestFormsApi2(Api2TestCase, FormsApiTestCase):
- def test_get_department_from_publisher_api(self):
- assert self.app.get('/api/2/util/publisher/2/department').body=='{"1": "National Health Service"}',self.app.get('/api/2/util/publisher/2/department').body
+ def test_get_department_from_organisation_api(self):
+ assert self.app.get('/api/2/util/publisher/2/department').body=='"7"',self.app.get('/api/2/util/publisher/2/department').body
class EmbeddedFormTestCase(BaseFormsApiCase, MockDrupalCase):
'''Tests the form as it would be used embedded in dgu html.'''
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckanext/dgu/tests/publishers/test_sync.py Fri Jul 15 18:03:56 2011 +0100
@@ -0,0 +1,54 @@
+from nose.tools import assert_equal
+
+from ckan import model
+from ckan.lib.create_test_data import CreateTestData
+
+from ckanext.dgu.publishers import sync
+from ckanext.dgu.drupalclient import DrupalClient
+
+from ckanext.dgu.tests import MockDrupalCase
+
+class TestSync(MockDrupalCase):
+ lots_of_publishers = True
+
+ def test_group_by_drupal_id(self):
+ CreateTestData.create_groups([
+ {'name': 'nhs',
+ 'drupal_id': '1'},
+ {'name': 'mod',
+ 'drupal_id': '2'},
+ {'name': 'london',
+ 'drupal_id': '3'},
+ ])
+ try:
+ group = sync.group_by_drupal_id(2)
+ assert group
+ assert_equal(group.name, 'mod')
+ finally:
+ model.repo.rebuild_db()
+
+ def test_sync_one(self):
+ groups = model.Session.query(model.Group)
+ assert groups.count() == 0
+
+ rev = model.repo.new_revision()
+ rev.author = 'okfn_maintenance'
+ rev.message = 'Syncing organisations.'
+
+ drupal_client = DrupalClient()
+ sync.sync_organisation(drupal_client, '16203') #HESA
+
+ model.repo.commit_and_remove()
+
+ groups = model.Session.query(model.Group)
+ assert_equal(groups.count(), 2)
+ group_hesa = model.Group.get(u'higher-education-statistics-agency')
+ group_bis = model.Group.get(u'department-for-business-innovation-and-skills')
+ assert group_hesa
+ assert group_bis
+ assert_equal(group_hesa.title, 'Higher Education Statistics Agency')
+ assert_equal(group_hesa.extras['drupal_id'], '16203')
+ assert_equal(group_hesa.extras['department_id'], group_bis.id)
+ assert_equal(group_bis.title, 'Department for Business, Innovation and Skills')
+ assert_equal(group_bis.extras['drupal_id'], '11399')
+ assert_equal(group_bis.extras['department_id'], group_bis.id)
--- a/setup.py Fri Jul 15 18:01:32 2011 +0100
+++ b/setup.py Fri Jul 15 18:03:56 2011 +0100
@@ -47,6 +47,7 @@
ons_delete_resourceless_packages = ckanext.dgu.bin.ons_delete_resourceless_packages:command
dump_analysis = ckanext.dgu.bin.dump_analysis:command
gov_daily = ckanext.dgu.bin.gov_daily:command
+ sync_organisations = ckanext.dgu.bin.sync_organisations:command
[ckan.forms]
package_gov3 = ckanext.dgu.forms.package_gov3:get_gov3_fieldset
http://bitbucket.org/okfn/ckanext-dgu/changeset/0ad06c03ad6f/
changeset: 0ad06c03ad6f
user: dread
date: 2011-07-15 19:08:32
summary: [merge]
affected #: 4 files (475 bytes)
--- a/ckanext/dgu/tests/forms/test_form_api.py Fri Jul 15 18:03:56 2011 +0100
+++ b/ckanext/dgu/tests/forms/test_form_api.py Fri Jul 15 18:08:32 2011 +0100
@@ -7,7 +7,10 @@
from ckan.tests import *
from ckan.tests import search_related
-from ckan.tests.functional.api.base import ApiTestCase, Api1TestCase, Api2TestCase, ApiUnversionedTestCase
+from ckan.tests.functional.api.base import (ApiTestCase,
+ Api1TestCase,
+ Api2TestCase,
+ ApiUnversionedTestCase)
from ckanext.harvest.lib import get_harvest_source, create_harvest_source
--- a/ckanext/dgu/tests/forms/test_form_api_gov3.py Fri Jul 15 18:03:56 2011 +0100
+++ b/ckanext/dgu/tests/forms/test_form_api_gov3.py Fri Jul 15 18:08:32 2011 +0100
@@ -9,9 +9,12 @@
from ckan.lib.helpers import literal
import ckan.model as model
from ckan.lib.create_test_data import CreateTestData
+
from test_form_api import BaseFormsApiCase
from ckanext.dgu.tests import MockDrupalCase, Gov3Fixtures, strip_organisation_id
-from ckan.tests.functional.api.base import ApiTestCase, Api1TestCase, Api2TestCase, ApiUnversionedTestCase
+from ckan.tests.functional.api.base import (Api1TestCase,
+ Api2TestCase,
+ ApiUnversionedTestCase)
class PackageFixturesBase:
--- a/ckanext/dgu/tests/functional/test_package_gov3.py Fri Jul 15 18:03:56 2011 +0100
+++ b/ckanext/dgu/tests/functional/test_package_gov3.py Fri Jul 15 18:08:32 2011 +0100
@@ -8,7 +8,10 @@
from ckan.lib.helpers import literal
from ckan.lib.create_test_data import CreateTestData
from ckanext.dgu.forms.formapi import DrupalRequestError
-from ckanext.dgu.tests.forms.test_form_api import BaseFormsApiCase, Api1TestCase, Api2TestCase
+from ckan.tests.functional.api.base import (Api1TestCase,
+ Api2TestCase,
+ ApiUnversionedTestCase)
+from ckanext.dgu.tests.forms.test_form_api import BaseFormsApiCase
from ckanext.dgu.tests import *
package_form = 'package_gov3'
--- a/requires/lucid_missing.txt Fri Jul 15 18:03:56 2011 +0100
+++ b/requires/lucid_missing.txt Fri Jul 15 18:08:32 2011 +0100
@@ -13,7 +13,9 @@
# ckanext-qa
-e hg+https://bitbucket.org/okfn/ckanext-qa@02c8c711c27f#egg=ckanext-qa
# ckanext-dgutheme
--e hg+https://bitbucket.org/okfn/ckanext-dgutheme@ba95c57574abegg=ckanext-dgutheme
+-e hg+https://bitbucket.org/okfn/ckanext-dgutheme@ba95c57574ab#egg=ckanext-dgutheme
+# ckanext-importlib
+-e hg+https://bitbucket.org/okfn/ckanext-importlib@7e314a51c4d6#egg=ckanext-importlib
# Use the OKFN fork of OWSlib (it has the tests/__init__.py file removed to remove
# a conflict with paste when installing as a debian package and it forces the use
# of lxml as the XML library).
Repository URL: https://bitbucket.org/okfn/ckanext-dgu/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list