[ckan-changes] commit/ckanext-storage: rgrp: [upload][m]: add support for uploading to local file storage as opposed to remote storage.
Bitbucket
commits-noreply at bitbucket.org
Thu May 26 10:57:08 UTC 2011
1 new changeset in ckanext-storage:
http://bitbucket.org/okfn/ckanext-storage/changeset/35cb57c3d542/
changeset: r27:35cb57c3d542
user: rgrp
date: 2011-05-26 12:57:00
summary: [upload][m]: add support for uploading to local file storage as opposed to remote storage.
* doc: also improve docs giving install instructions, more detail of config options etc
* setup.py: bump version to 0.5a
affected #: 6 files (2.9 KB)
--- a/.hgignore Sat May 14 17:07:15 2011 +0100
+++ b/.hgignore Thu May 26 11:57:00 2011 +0100
@@ -4,4 +4,5 @@
*.egg-info/*
sandbox/*
.DS_Store
+build/*
--- a/README.rst Sat May 14 17:07:15 2011 +0100
+++ b/README.rst Thu May 26 11:57:00 2011 +0100
@@ -10,17 +10,46 @@
It uses `OFS`_ to talk to the backing storage so can support anything that OFS
supports including local filesytem, S3, Google Storage etc.
-.. _OFS: http://pypi.python.org/pypi/ofs/
+.. _OFS: http://packages.python.org/ofs/
+
+Installation
+============
+
+Install the extension::
+
+ # using pip (could use easy_install)
+ pip install ckanext-storage
+ # could install from source
+ # hg clone https://bitbucket.org/okfn/ckanext-storage
+ # cd ckanext-storage
+ # pip install -e .
+
+Note that for use of S3-like backends (S3, Google Storage etc) you will need boto (this is installed by default at the moment). For local filesystem backend you need to install pairtree (`pip install pairtree`).
In your config you need something like::
ckan.plugins = storage
- # this is for google storage
+
+ ## OFS configuration
+ ## This is for google storage. Example for another backend is below
+ ## See OFS docs for full details
ofs.impl = google
ofs.gs_access_key_id = GOOGCABCDASDASD
ofs.gs_secret_access_key = 134zsdfjkw4234addad
- ckanext.storage.bucket = the bucket to use for uploading
- ckanext.storage.max_content_length = [optional] maximum content size for uploads (defaults to 50Mb)
+ ## bucket to use in storage You *must* set this
+ ckanext.storage.bucket = ....
+
+ ## optional
+ ## maximum content size for uploads in bytes, defaults to 1Gb
+ # ckanext.storage.max_content_length = 1000000000
+ ## prefix for all keys. Useful because we may use a single bucket and want to
+ ## partition file uploads. Defaults to file/
+ # ckanext.storage.key_prefix = file/
+
+For local file storage you would replace ofs arguments with::
+
+ ofs.impl = pairtree
+ ofs.storage_dir = /my/path/to/storage/root/directory
Upload Web Interface
--- a/ckanext/storage/__init__.py Sat May 14 17:07:15 2011 +0100
+++ b/ckanext/storage/__init__.py Thu May 26 11:57:00 2011 +0100
@@ -30,7 +30,10 @@
# upload page
route_map.connect('storage_upload', '/storage/upload',
controller='ckanext.storage.controller:StorageController',
- action='index')
+ action='upload')
+ route_map.connect('storage_upload_handle', '/storage/upload_handle',
+ controller='ckanext.storage.controller:StorageController',
+ action='upload_handle')
route_map.connect('storage_upload_success', '/storage/upload/success',
controller='ckanext.storage.controller:StorageController',
action='success')
--- a/ckanext/storage/controller.py Sat May 14 17:07:15 2011 +0100
+++ b/ckanext/storage/controller.py Thu May 26 11:57:00 2011 +0100
@@ -1,5 +1,6 @@
import re
from datetime import datetime
+from cgi import FieldStorage
try:
from cStringIO import StringIO
except ImportError:
@@ -16,6 +17,7 @@
from pylons import request, response
from pylons.controllers.util import abort, redirect_to
from pylons import config
+from paste.fileapp import FileApp
from ckan.lib.base import BaseController, c, request, render, config, h, abort
from ckan.lib.jsonp import jsonpify
@@ -251,15 +253,14 @@
'''
ofs = get_ofs()
- def index(self):
- label = key_prefix + request.params.get('filepath', str(uuid.uuid4()))
+ def _get_form_for_remote(self):
# would be nice to use filename of file
# problem is 'we' don't know this at this point and cannot add it to
# success_action_redirect and hence cannnot display to user afterwards
# + '/${filename}'
+ label = key_prefix + request.params.get('filepath', str(uuid.uuid4()))
method = 'POST'
authorize(method, BUCKET, label, c.userobj, self.ofs)
-
content_length_range = int(
config.get('ckanext.storage.max_content_length',
50000000))
@@ -277,7 +278,7 @@
c.data = self.ofs.conn.build_post_form_args(
BUCKET,
label,
- expires_in=600,
+ expires_in=3600,
max_content_length=content_length_range,
success_action_redirect=success_action_redirect,
acl=acl,
@@ -293,8 +294,43 @@
if field['name'] == 'content-length-range':
del c.data['fields'][idx]
c.data_json = json.dumps(c.data, indent=2)
+
+ def upload(self):
+ if storage_backend in ['google', 's3']:
+ self._get_form_for_remote()
+ else:
+ label = key_prefix + request.params.get('filepath', str(uuid.uuid4()))
+ c.data = {
+ 'action': h.url_for('storage_upload_handle'),
+ 'fields': [
+ {
+ 'name': 'key',
+ 'value': label
+ }
+ ]
+ }
return render('ckanext/storage/index.html')
+ def upload_handle(self):
+ bucket_id = BUCKET
+ params = request.params
+ params = dict(params.items())
+ stream = params.get('file')
+ label = params.get('key')
+ authorize('POST', BUCKET, label, c.userobj, self.ofs)
+ if not label:
+ abort(400, "No label")
+ if not isinstance(stream, FieldStorage):
+ abort(400, "No file stream.")
+ del params['file']
+ params['filename-original'] = stream.filename
+ params['_owner'] = c.userobj.id
+ params['uploaded-by'] = c.userobj.id
+ self.ofs.put_stream(bucket_id, label, stream.file, params)
+ success_action_redirect = h.url_for('storage_upload_success', qualified=True,
+ bucket=BUCKET, label=label)
+ h.redirect_to(success_action_redirect)
+
def success(self):
h.flash_success('Upload successful')
c.file_url = h.url_for('storage_file',
@@ -305,6 +341,15 @@
return render('ckanext/storage/success.html')
def file(self, label):
- url = self.ofs.get_url(BUCKET, label)
- h.redirect_to(url)
+ file_url = self.ofs.get_url(BUCKET, label)
+ if file_url.startswith("file://"):
+ metadata = self.ofs.get_metadata(BUCKET, label)
+ filepath = file_url[len("file://"):]
+ headers = {'Content-Disposition':'attachment; filename="%s"' %
+ label,
+ 'Content-Type':metadata.get('_format', 'text/plain')}
+ fapp = FileApp(filepath, headers=None, **headers)
+ return fapp(request.environ, self.start_response)
+ else:
+ h.redirect_to(file_url)
--- a/ckanext/storage/templates/ckanext/storage/index.html Sat May 14 17:07:15 2011 +0100
+++ b/ckanext/storage/templates/ckanext/storage/index.html Thu May 26 11:57:00 2011 +0100
@@ -14,8 +14,8 @@
<div py:match="content"><h1>Upload</h1>
- <p>This upload form is valid for 10m. Please reload the page after 10m to
- start again.</p>
+ <p>This upload form is valid for a limited time (usually 1h or so). If the
+ form expires please reload the page.</p><form action="${c.data['action']}"
enctype="multipart/form-data"
--- a/setup.py Sat May 14 17:07:15 2011 +0100
+++ b/setup.py Thu May 26 11:57:00 2011 +0100
@@ -1,7 +1,7 @@
from setuptools import setup, find_packages
import sys, os
-version = '0.4'
+version = '0.5a'
try:
long_description = open('README.txt').read()
except:
Repository URL: https://bitbucket.org/okfn/ckanext-storage/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list