[ckan-changes] commit/ckanextiati: 6 new changesets
Bitbucket
commits-noreply at bitbucket.org
Mon Oct 24 13:58:21 UTC 2011
6 new commits in ckanextiati:
https://bitbucket.org/okfn/ckanextiati/changeset/14ad0472f3a4/
changeset: 14ad0472f3a4
branch: spreadsheet-support
user: amercader
date: 2011-10-21 13:05:32
summary: Add template for file upload
affected #: 1 file
diff -r a6a2bf03cf40b91e319e85ee672762250538780e -r 14ad0472f3a4af828a88f9b40827628094771042 ckanext/iati/templates/csv/upload.html
--- /dev/null
+++ b/ckanext/iati/templates/csv/upload.html
@@ -0,0 +1,27 @@
+<html xmlns:py="http://genshi.edgewall.org/"
+ xmlns:i18n="http://genshi.edgewall.org/i18n"
+ xmlns:xi="http://www.w3.org/2001/XInclude"
+ py:strip="">
+ <py:def function="page_title">CSV Import</py:def>
+ <div py:match="content">
+
+ <h2 class="head">CSV Import</h2>
+
+ <div>Please provide a CSV file.</div>
+ <div>Template?</div>
+ <hr class="cleared" />
+ <form id="csv-upload" action="/csv/upload" method="POST" enctype="multipart/form-data">
+ <dl>
+ <dt><label for="file">CSV file</label></dt>
+ <dd><input type="file" id="file" name="file"/></dd>
+ </dl>
+ <div class="submit">
+ <input id="upload" name="upload" type="submit" value="Upload" />
+ </div>
+
+ </form>
+
+ </div>
+
+ <xi:include href="../layout.html" />
+</html>
https://bitbucket.org/okfn/ckanextiati/changeset/f3eca52591ec/
changeset: f3eca52591ec
branch: spreadsheet-support
user: amercader
date: 2011-10-21 16:26:58
summary: Add results page for import
affected #: 2 files
diff -r 14ad0472f3a4af828a88f9b40827628094771042 -r f3eca52591ec15fa26d0860af9ffbec011fceef2 ckanext/iati/controllers/spreadsheet.py
--- a/ckanext/iati/controllers/spreadsheet.py
+++ b/ckanext/iati/controllers/spreadsheet.py
@@ -31,7 +31,7 @@
('verification-status','extras', 'verified'),
('default-language','extras', 'language')
]
-
+
def __before__(self, action, **params):
super(CSVController,self).__before__(action, **params)
@@ -46,7 +46,7 @@
def download(self,publisher=None):
context = {'model':model,'user': c.user or c.author}
-
+
if publisher and publisher != 'all':
try:
group = get_action('group_show')(context, {'id':publisher})
@@ -95,13 +95,16 @@
return render('csv/upload.html')
elif request.method == 'POST':
csv_file = request.POST['file']
-
+ c.file_name = csv_file.filename
+
added, updated, errors = self.read_csv_file(csv_file)
c.added = added
c.updated = updated
c.errors = errors
- return 'Packages added: %i, Packages updated: %i, Errors: %s' % \
- (added,updated,', '.join(errors))
+
+ return render('csv/result.html')
+ return 'Packages added: %s<br/> Packages updated: %s<br/> Errors: %s<br/>' % \
+ ('<br/> '.join(added),'<br/> '.join(updated),'<br/> '.join(errors))
def write_csv_file(self,publisher):
context = {'model':model,'user': c.user or c.author}
@@ -157,16 +160,16 @@
#TODO: separator
reader = csv.DictReader(csv_file.file)
- counts = {'added': 0, 'updated': 0}
+ counts = {'added': [], 'updated': []}
errors = []
for i,row in enumerate(reader):
try:
# Check mandatory fields
if not row['registry-publisher-id']:
raise ValueError('Publisher not defined')
-
+
# TODO: Check permissions on group
-
+
if not row['registry-file-id']:
raise ValueError('File id not defined')
# TODO: Check name convention
@@ -221,13 +224,13 @@
package_dict.update({'id':existing_package_dict['id']})
updated_package = get_action('package_update_rest')(context, package_dict)
if counts:
- counts['updated'] += 1
+ counts['updated'].append(updated_package['name'])
except NotFound:
# Package needs to be created
log.info('Package with name "%s" does not exist and will be created' % package_dict['name'])
new_package = get_action('package_create_rest')(context, package_dict)
if counts:
- counts['added'] += 1
+ counts['added'].append(new_package['name'])
except ValidationError,e:
raise ValueError(str(e))
diff -r 14ad0472f3a4af828a88f9b40827628094771042 -r f3eca52591ec15fa26d0860af9ffbec011fceef2 ckanext/iati/templates/csv/result.html
--- /dev/null
+++ b/ckanext/iati/templates/csv/result.html
@@ -0,0 +1,42 @@
+<html xmlns:py="http://genshi.edgewall.org/"
+ xmlns:i18n="http://genshi.edgewall.org/i18n"
+ xmlns:xi="http://www.w3.org/2001/XInclude"
+ py:strip="">
+ <py:def function="page_title">CSV Import Results</py:def>
+ <div py:match="content">
+
+ <h2 class="head">CSV Import Results</h2>
+
+ <div>CSV file <strong>${c.file_name}</strong> imported.</div>
+ <hr class="cleared" />
+ <h3>Summary</h3>
+ <ul>
+ <li><a href="#added">Datasets created: ${len(c.added)}</a></li>
+ <li><a href="#updated">Datasets updated: ${len(c.updated)}</a></li>
+ <li><a href="#errors">Errors found: ${len(c.errors)}</a></li>
+ </ul>
+ <h3><a name="added">Datasets added</a></h3>
+ <ul>
+ <py:for each="pkg in c.added">
+ <li><a href="${h.url_for(controller='package', action='read', id=pkg)}">${g.site_url}${h.url_for(controller='package', action='read', id=pkg)}</a></li>
+ </py:for>
+ </ul>
+
+ <h3><a name="updated">Datasets updated</a></h3>
+ <ul>
+ <py:for each="pkg in c.updated">
+ <li><a href="${h.url_for(controller='package', action='read', id=pkg)}">${g.site_url}${h.url_for(controller='package', action='read', id=pkg)}</a></li>
+ </py:for>
+ </ul>
+
+ <h3><a name="errors">Errors found</a></h3>
+ <ul>
+ <py:for each="error in c.errors">
+ <li>${error}</li>
+ </py:for>
+ </ul>
+
+ </div>
+
+ <xi:include href="../layout.html" />
+</html>
https://bitbucket.org/okfn/ckanextiati/changeset/e457361affec/
changeset: e457361affec
branch: spreadsheet-support
user: amercader
date: 2011-10-24 13:09:15
summary: Only fields present in the spreadsheet will be updated
affected #: 1 file
diff -r f3eca52591ec15fa26d0860af9ffbec011fceef2 -r e457361affec649c31e39664288a9d1664c9d537 ckanext/iati/controllers/spreadsheet.py
--- a/ckanext/iati/controllers/spreadsheet.py
+++ b/ckanext/iati/controllers/spreadsheet.py
@@ -168,8 +168,6 @@
if not row['registry-publisher-id']:
raise ValueError('Publisher not defined')
- # TODO: Check permissions on group
-
if not row['registry-file-id']:
raise ValueError('File id not defined')
# TODO: Check name convention
@@ -186,20 +184,21 @@
def get_package_dict_from_row(self,row):
package = {}
for fieldname, entity, key in self.csv_mapping:
- value = row[fieldname]
- if value:
- if entity == 'groups':
- package['groups'] = [value]
- elif entity == 'resources':
- if not 'resources' in package:
- package['resources'] = [{}]
- package['resources'][0][key] = value
- elif entity == 'extras':
- if not 'extras' in package:
- package['extras'] = {}
- package['extras'][key] = value
- else:
- package[key] = value
+ if fieldname in row:
+ value = row[fieldname]
+ if value:
+ if entity == 'groups':
+ package['groups'] = [value]
+ elif entity == 'resources':
+ if not 'resources' in package:
+ package['resources'] = [{}]
+ package['resources'][0][key] = value
+ elif entity == 'extras':
+ if not 'extras' in package:
+ package['extras'] = {}
+ package['extras'][key] = value
+ else:
+ package[key] = value
return package
def create_or_update_package(self, package_dict, counts = None):
https://bitbucket.org/okfn/ckanextiati/changeset/21fdeac8a1db/
changeset: 21fdeac8a1db
branch: spreadsheet-support
user: amercader
date: 2011-10-24 13:35:03
summary: Check name convention on new packages and extra log messages
affected #: 1 file
diff -r e457361affec649c31e39664288a9d1664c9d537 -r 21fdeac8a1db3a9e8d35d1a495610c654fca8e19 ckanext/iati/controllers/spreadsheet.py
--- a/ckanext/iati/controllers/spreadsheet.py
+++ b/ckanext/iati/controllers/spreadsheet.py
@@ -102,9 +102,10 @@
c.updated = updated
c.errors = errors
+ log.info('CSV export finished: file %s, %i added, %i updated, %i errors' % \
+ (c.file_name,len(c.added),len(c.updated),len(c.errors)))
+
return render('csv/result.html')
- return 'Packages added: %s<br/> Packages updated: %s<br/> Errors: %s<br/>' % \
- ('<br/> '.join(added),'<br/> '.join(updated),'<br/> '.join(errors))
def write_csv_file(self,publisher):
context = {'model':model,'user': c.user or c.author}
@@ -160,6 +161,9 @@
#TODO: separator
reader = csv.DictReader(csv_file.file)
+ context = {'model':model,'user': c.user or c.author, 'api_verion':'1'}
+ groups= get_action('group_list')(context, {})
+
counts = {'added': [], 'updated': []}
errors = []
for i,row in enumerate(reader):
@@ -170,14 +174,24 @@
if not row['registry-file-id']:
raise ValueError('File id not defined')
- # TODO: Check name convention
+
+ # Check name convention
+ name = row['registry-file-id']
+ parts = name.split('-')
+ group_name = parts[0] if len(parts) == 2 else '-'.join(parts[:-1])
+ if not group_name or not group_name in groups:
+ raise ValueError('Dataset name does not follow the convention <publisher>-<code>: "%s"' % group_name)
package_dict = self.get_package_dict_from_row(row)
self.create_or_update_package(package_dict,counts)
except ValueError,e:
- errors.append('Error in row %i: %s' % (i+1,str(e)))
+ msg = 'Error in row %i: %s' % (i+1,str(e))
+ log.error(msg)
+ errors.append(msg)
except NotAuthorized,e:
- errors.append('Error in row %i: Not authorized to publish to this group (%s)' % (i+1,row['registry-publisher-id']))
+ msg = 'Error in row %i: Not authorized to publish to this group: %s' % (i+1,row['registry-publisher-id'])
+ log.error(msg)
+ errors.append(msg)
return counts['added'], counts['updated'], errors
@@ -216,21 +230,23 @@
data_dict['id'] = package_dict['name']
try:
existing_package_dict = get_action('package_show')(context, data_dict)
+
# Update package
+ log.info('Package with name "%s" exists and will be updated' % package_dict['name'])
- log.info('Package with name "%s" exists and will be updated' % package_dict['name'])
context.update({'id':existing_package_dict['id']})
package_dict.update({'id':existing_package_dict['id']})
updated_package = get_action('package_update_rest')(context, package_dict)
if counts:
counts['updated'].append(updated_package['name'])
+ log.debug('Package with name "%s" updated' % package_dict['name'])
except NotFound:
# Package needs to be created
log.info('Package with name "%s" does not exist and will be created' % package_dict['name'])
new_package = get_action('package_create_rest')(context, package_dict)
if counts:
counts['added'].append(new_package['name'])
-
+ log.debug('Package with name "%s" created' % package_dict['name'])
except ValidationError,e:
raise ValueError(str(e))
https://bitbucket.org/okfn/ckanextiati/changeset/5d40356f6c75/
changeset: 5d40356f6c75
branch: spreadsheet-support
user: amercader
date: 2011-10-24 13:47:48
summary: Extra authz check when uploading
affected #: 1 file
diff -r 21fdeac8a1db3a9e8d35d1a495610c654fca8e19 -r 5d40356f6c7587c75e759cf67ee7ce0f24b7013c ckanext/iati/controllers/spreadsheet.py
--- a/ckanext/iati/controllers/spreadsheet.py
+++ b/ckanext/iati/controllers/spreadsheet.py
@@ -91,6 +91,10 @@
return output
def upload(self):
+ if not self.is_sysadmin and not self.authz_groups:
+ # User does not have permissions on any publisher
+ abort(403,'Permission denied')
+
if request.method == 'GET':
return render('csv/upload.html')
elif request.method == 'POST':
https://bitbucket.org/okfn/ckanextiati/changeset/becb06113202/
changeset: becb06113202
branch: spreadsheet-support
user: amercader
date: 2011-10-24 14:46:27
summary: Sniff CSV file dialect to support different delimiters, etc.
affected #: 1 file
diff -r 5d40356f6c7587c75e759cf67ee7ce0f24b7013c -r becb0611320218244bf719ce15d3f3db077a642b ckanext/iati/controllers/spreadsheet.py
--- a/ckanext/iati/controllers/spreadsheet.py
+++ b/ckanext/iati/controllers/spreadsheet.py
@@ -106,7 +106,7 @@
c.updated = updated
c.errors = errors
- log.info('CSV export finished: file %s, %i added, %i updated, %i errors' % \
+ log.info('CSV import finished: file %s, %i added, %i updated, %i errors' % \
(c.file_name,len(c.added),len(c.updated),len(c.errors)))
return render('csv/result.html')
@@ -161,9 +161,14 @@
def read_csv_file(self,csv_file):
fieldnames = [n[0] for n in self.csv_mapping]
- #reader = csv.DictReader(csv_file.file,fieldnames=fieldnames)
- #TODO: separator
- reader = csv.DictReader(csv_file.file)
+
+ # Try to sniff the file dialect
+ dialect = csv.Sniffer().sniff(csv_file.file.read(1024))
+ csv_file.file.seek(0)
+ reader = csv.DictReader(csv_file.file, dialect=dialect)
+
+ log.debug('Starting reading file %s (delimiter "%s", escapechar "%s")' %
+ (csv_file.filename,dialect.delimiter,dialect.escapechar))
context = {'model':model,'user': c.user or c.author, 'api_verion':'1'}
groups= get_action('group_list')(context, {})
Repository URL: https://bitbucket.org/okfn/ckanextiati/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list