[ckan-changes] commit/ckanext-pdeu: 2 new changesets
Bitbucket
commits-noreply at bitbucket.org
Thu Jun 9 17:46:11 UTC 2011
2 new changesets in ckanext-pdeu:
http://bitbucket.org/okfn/ckanext-pdeu/changeset/40d78822912c/
changeset: 40d78822912c
user: pudo
date: 2011-06-09 17:42:50
summary: fix up footer with languages, various other links, lod2 logo
affected #: 3 files (8.9 KB)
--- a/ckanext/pdeu/theme/public/css/style.css Thu Jun 09 16:46:25 2011 +0200
+++ b/ckanext/pdeu/theme/public/css/style.css Thu Jun 09 17:42:50 2011 +0200
@@ -98,10 +98,18 @@
padding-top: 1em;
}
+footer a {
+ text-decoration: none;
+}
+
footer #footer-okf-logo {
margin-bottom: -4px;
}
+footer .xoxo h3 {
+ font-size: 1.2em;
+}
+
/* Basic page elements */
Binary file ckanext/pdeu/theme/public/img/lod2.png has changed
--- a/ckanext/pdeu/theme/templates/layout_base.html Thu Jun 09 16:46:25 2011 +0200
+++ b/ckanext/pdeu/theme/templates/layout_base.html Thu Jun 09 17:42:50 2011 +0200
@@ -110,14 +110,72 @@
</div><div class="footer outer"><footer class="container">
- <p id="credits">
+ <div class="xoxo span-6">
+ <h3 class="widget-title">About ${g.site_title}</h3>
+ <div class="textwidget">
+ <ul>
+ <li>${h.link_to(_('About'), h.url_for(controller='home', action='about', id=None))}</li>
+ <li>
+ <a href="http://lod2.okfn.org/">Blog</a>
+ </li>
+ <li>${h.link_to(_('API'), h.url_for(controller='api', action='get_api', id=None))}</li>
+ <li>${h.link_to(_('API Docs'), 'http://wiki.ckan.net/API')}</li>
+ <li>
+ <a href="http://www.okfn.org/contact/">Contact Us</a>
+ </li>
+ <li>
+ <a href="http://www.okfn.org/privacy-policy/">Privacy Policy</a>
+ </li>
+ </ul>
+ </div>
+ </div>
+ <div class="xoxo span-6">
+ <h3 class="widget-title">Languages</h3>
+ <div class="textwidget">
+ <ul>
+<?python
+ import ckan.lib.hash
+ current_url = url.current().encode('utf-8')
+ current_url_hash = ckan.lib.hash.get_message_hash(current_url)
+?>
+ <li py:for="locale in h.get_available_locales()[0::2]">
+ <a href="${url(controller='home', action='locale',
+ locale=str(locale),
+ return_to=current_url,
+ hash=current_url_hash)}">
+ ${locale.display_name or locale.english_name}
+ </a>
+ </li>
+ </ul>
+ </div>
+ </div>
+ <div class="xoxo span-6">
+ <h3 class="widget-title"> </h3>
+ <div class="textwidget">
+ <ul>
+ <li py:for="locale in h.get_available_locales()[1::2]">
+ <a href="${url(controller='home', action='locale',
+ locale=str(locale),
+ return_to=current_url,
+ hash=current_url_hash)}">
+ ${locale.display_name or locale.english_name}
+ </a>
+ </li>
+ </ul>
+ </div>
+ </div>
+ <div class="xoxo span-6 last">
+ <h3 class="widget-title"> </h3>
+ <p id="credits">
+ © 2011:
<img src="http://assets.okfn.org/images/logo/okf_logo_white_and_green_tiny.png" id="footer-okf-logo" />
- An <a href="http://www.okfn.org/">Open Knowledge Foundation</a> Project © 2011
- <a href="http://www.opendefinition.org/okd/">
- <img alt="This Content and Data is Open" src="http://assets.okfn.org/images/ok_buttons/od_80x15_blue.png" style="border: none ; margin-bottom: -4px;"/></a>
- Based on the open source <a
- href="http://ckan.org">CKAN</a> data management platform.
+ <a href="http://www.okfn.org/">Open Knowledge Foundation</a><br/><br/>
+ Based on the <a
+ href="http://ckan.org">CKAN</a> data management platform.<br/>
+ <a href="http://www.opendefinition.org/okd/"><img alt="This Content and Data is Open" src="http://assets.okfn.org/images/ok_buttons/od_80x15_blue.png" style="border: none ; margin-bottom: -4px;"/></a></p>
+ <a href="http://lod2.eu"><img src="/img/lod2.png" alt="Part of the LOD2 Project" /></a>
+ </div></footer></div><!-- eo #container -->
http://bitbucket.org/okfn/ckanext-pdeu/changeset/d412a1a2859c/
changeset: d412a1a2859c
user: pudo
date: 2011-06-09 19:45:53
summary: scrape dati.piemonte.it
affected #: 3 files (5.4 KB)
--- a/ckanext/pdeu/harvesters/__init__.py Thu Jun 09 17:42:50 2011 +0200
+++ b/ckanext/pdeu/harvesters/__init__.py Thu Jun 09 19:45:53 2011 +0200
@@ -6,5 +6,6 @@
from vienna import DataWienGvAtHarvester
from paris import OpendataParisFrHarvester
from digitaliser_dk import DigitaliserDkHarvester
+from piemonte import DatiPiemonteItHarvester
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckanext/pdeu/harvesters/piemonte.py Thu Jun 09 19:45:53 2011 +0200
@@ -0,0 +1,135 @@
+#coding: utf-8
+from datetime import datetime
+from itertools import count
+from urlparse import urljoin
+from urllib2 import urlopen
+import os
+from lxml import html
+import logging
+from hashlib import sha1
+
+from ckan.lib.helpers import json
+
+from ckanext.harvest.model import HarvestObject
+from ckanext.harvest.harvesters import HarvesterBase
+
+log = logging.getLogger(__name__)
+
+class DatiPiemonteItHarvester(HarvesterBase):
+ INDEX_URL = "http://dati.piemonte.it/dati.html?start=%s&limit=%s"
+
+ def info(self):
+ return {
+ 'name': 'dati_piemonte_it',
+ 'title': 'Dati Piemonte',
+ 'description': ''
+ }
+
+ def gather_stage(self, harvest_job):
+ log.debug('In DatiPiemonteIt gather_stage')
+
+ step = 100
+ ids = []
+ links = []
+ for i in count(1):
+ doc = html.parse(self.INDEX_URL % (step, step*i))
+ for link in doc.findall("//div[@class='datiItem']/a"):
+ link = urljoin(self.INDEX_URL, link.get('href'))
+ if link in links:
+ return ids
+ links.append(link)
+ id = sha1(link.encode('utf-8')).hexdigest()
+ obj = HarvestObject(guid=id, job=harvest_job, content=link)
+ obj.save()
+ ids.append(obj.id)
+ return ids
+
+ def fetch_stage(self, harvest_object):
+ doc = html.parse(harvest_object.content)
+ package_dict = {'extras': {}, 'resources': [], 'tags': []}
+ package_dict['title'] = doc.findtext('//h2[@class="itemTitle"]').strip()
+ package_dict['notes'] = doc.find('//div[@class="itemFullText"]').xpath('string()').strip()
+ source = doc.find('//div[@class="itemFonte"]/a')
+ if source is not None:
+ package_dict['url'] = source.get('href')
+ package_dict['author'] = doc.find('//div[@class="itemFonte"]').xpath('string()')\
+ .replace('Fonte:', '').strip()
+ package_dict['extras']['harvest_dataset_url'] = harvest_object.content
+ file_type = ''
+ for block in doc.findall('//div[@class="itemBlock"]'):
+ name = block.findtext('h3')
+ if 'Tipo di file' in name:
+ file_type = block.find('p').xpath('string()')
+ elif 'Scala' in name:
+ package_dict['extras']['scale'] = \
+ block.find('p').xpath('string()')
+ elif 'Sistema di riferimento' in name:
+ package_dict['extras']['reference_system'] = \
+ block.find('p').xpath('string()')
+ elif 'Frequenza di aggiornamento' in name:
+ package_dict['extras']['temporal_granularity'] = \
+ block.find('p').xpath('string()').strip()
+ elif 'Data aggiornamento della scheda' in name:
+ package_dict['metadata_modified'] = \
+ block.find('p').xpath('string()').strip()
+ elif 'Data aggiornamento del dato' in name:
+ package_dict['metadata_created'] = \
+ block.find('p').xpath('string()').strip()
+ elif 'Ente proprietario' in name:
+ if block.find('h3').tail:
+ package_dict['maintainer'] = \
+ block.find('h3').tail.strip()
+ elif 'Tag' in name:
+ for a in block.findall('.//a'):
+ package_dict['tags'].append(a.text)
+ elif 'Argomenti' in name:
+ package_dict['extras']['categories'] = []
+ for span in block.findall('span'):
+ if span.tail:
+ package_dict['extras']['categories'].append(span.tail)
+
+ downloadForm = doc.find("//form[@id='downloadForm']")
+ if downloadForm:
+ _dl = urljoin(self.INDEX_URL, downloadForm.get('action'))
+ fh = urlopen(_dl)
+ package_dict['resources'].append({
+ 'url': fh.url,
+ 'format': file_type,
+ 'description': os.path.basename(fh.url)
+ })
+ fh.close()
+ #from pprint import pprint
+ #pprint(package_dict)
+ #package_dict['license_id'] = 'odc-odbl'
+ harvest_object.content = json.dumps(package_dict)
+ harvest_object.save()
+ return True
+
+ def import_stage(self,harvest_object):
+ if not harvest_object:
+ log.error('No harvest object received')
+ return False
+
+ if harvest_object.content is None:
+ self._save_object_error('Empty content for object %s' % harvest_object.id,harvest_object,'Import')
+ return False
+
+ try:
+ package_dict = json.loads(harvest_object.content)
+ package_dict['id'] = harvest_object.guid
+ package_dict['name'] = self._gen_new_name(package_dict['title'])
+
+ # Common extras
+ package_dict['extras']['harvest_catalogue_name'] = u'Dati Piemonte'
+ package_dict['extras']['harvest_catalogue_url'] = u'http://dati.piemonte.it/'
+ package_dict['extras']['eu_country'] = u'IT'
+ package_dict['extras']['eu_nuts1'] = u'ITC'
+ package_dict['extras']['eu_nuts2'] = u'ITC1'
+
+ return self._create_or_update_package(package_dict, harvest_object)
+ except Exception, e:
+ log.exception(e)
+ self._save_object_error('%r' % e, harvest_object, 'Import')
+
+
+
--- a/setup.py Thu Jun 09 17:42:50 2011 +0200
+++ b/setup.py Thu Jun 09 19:45:53 2011 +0200
@@ -34,5 +34,6 @@
data_wien_gv_at_harvester=ckanext.pdeu.harvesters:DataWienGvAtHarvester
opendata_paris_fr_harvester=ckanext.pdeu.harvesters:OpendataParisFrHarvester
digitaliser_dk_harvester=ckanext.pdeu.harvesters:DigitaliserDkHarvester
+ piemonte_harvester=ckanext.pdeu.harvesters:DatiPiemonteItHarvester
""",
)
Repository URL: https://bitbucket.org/okfn/ckanext-pdeu/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list