[ckan-changes] commit/ckan: 5 new changesets
Bitbucket
commits-noreply at bitbucket.org
Thu Jun 16 11:03:25 UTC 2011
5 new changesets in ckan:
http://bitbucket.org/okfn/ckan/changeset/4c732f61091a/
changeset: 4c732f61091a
branch: feature-1141-moderated-edits-ajax
user: kindly
date: 2011-06-15 11:08:24
summary: [controller] view package at any date
affected #: 2 files (427 bytes)
--- a/ckan/controllers/package.py Tue Jun 14 12:03:01 2011 +0100
+++ b/ckan/controllers/package.py Wed Jun 15 10:08:24 2011 +0100
@@ -2,6 +2,8 @@
import urlparse
from urllib import urlencode
import json
+import datetime
+import re
from sqlalchemy.orm import eagerload_all
from sqlalchemy import or_
@@ -174,11 +176,18 @@
@proxy_cache()
def read(self, id):
-
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'schema': self._form_to_db_schema(),
'id': id}
+ split = id.split('@')
+ if len(split) == 2:
+ context['id'], revision = split
+ try:
+ date = datetime.datetime(*map(int, re.split('[^\d]', revision)))
+ context['revision_date'] = date
+ except ValueError:
+ context['revision_id'] = revision
#check if package exists
try:
c.pkg_dict = get.package_show(context)
@@ -187,7 +196,6 @@
abort(404, _('Package not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % id)
-
cache_key = self._pkg_cache_key(c.pkg)
etag_cache(cache_key)
--- a/ckan/lib/dictization/model_dictize.py Tue Jun 14 12:03:01 2011 +0100
+++ b/ckan/lib/dictization/model_dictize.py Wed Jun 15 10:08:24 2011 +0100
@@ -5,6 +5,7 @@
from ckan.lib.dictization import (obj_list_dictize,
obj_dict_dictize,
table_dictize)
+from ckan.logic import NotFound
import ckan.misc
import json
@@ -102,6 +103,8 @@
package_rev = model.package_revision_table
q = select([package_rev]).where(package_rev.c.id == pkg.id)
result = _execute_with_revision(q, package_rev, context).first()
+ if not result:
+ raise NotFound
result_dict = table_dictize(result, context)
#resources
res_rev = model.resource_revision_table
http://bitbucket.org/okfn/ckan/changeset/dcc16659205f/
changeset: dcc16659205f
branch: feature-1141-moderated-edits-ajax
user: kindly
date: 2011-06-15 14:32:43
summary: [home] only current displayed in list
affected #: 6 files (3.5 KB)
--- a/ckan/authz.py Wed Jun 15 10:08:24 2011 +0100
+++ b/ckan/authz.py Wed Jun 15 13:32:43 2011 +0100
@@ -184,7 +184,6 @@
user = model.User.by_name(username, autoflush=False)
else:
user = None
- entity.roles.property.mapper.class_
visitor = model.User.by_name(model.PSEUDO_USER__VISITOR, autoflush=False)
logged_in = model.User.by_name(model.PSEUDO_USER__LOGGED_IN,
autoflush=False)
@@ -193,8 +192,15 @@
# need to use this in the queries below as if we use
# model.UserObjectRole a cross join happens always
# returning all the roles.
- role_cls = entity.roles.property.mapper.class_
- q = q.outerjoin('roles')
+ if hasattr(entity, 'continuity'):
+ q = q.filter_by(current=True)
+ q = q.outerjoin('continuity', 'roles')
+ continuity = entity.continuity.property.mapper.class_
+ role_cls = continuity.roles.property.mapper.class_
+ else:
+ role_cls = entity.roles.property.mapper.class_
+ q = q.outerjoin('roles')
+
if hasattr(entity, 'state'):
state = entity.state
else:
--- a/ckan/controllers/home.py Wed Jun 15 10:08:24 2011 +0100
+++ b/ckan/controllers/home.py Wed Jun 15 13:32:43 2011 +0100
@@ -5,6 +5,7 @@
from genshi.template import NewTextTemplate
from ckan.authz import Authorizer
+from ckan.logic.action.get import current_package_list_with_resources
from ckan.i18n import set_session_locale
from ckan.lib.search import query_for, QueryOptions, SearchError
from ckan.lib.cache import proxy_cache, get_cache_expires
@@ -46,10 +47,9 @@
c.facets = query.facets
c.fields = []
c.package_count = query.count
- c.latest_packages = self.authorizer.authorized_query(c.user, model.Package)\
- .join('revision').order_by(model.Revision.timestamp.desc())\
- .limit(5).all()
-
+ c.latest_packages = current_package_list_with_resources({'model': model,
+ 'user': c.user,
+ 'limit': 5})
return render('home/index.html', cache_key=cache_key,
cache_expire=cache_expires)
--- a/ckan/lib/dictization/__init__.py Wed Jun 15 10:08:24 2011 +0100
+++ b/ckan/lib/dictization/__init__.py Wed Jun 15 13:32:43 2011 +0100
@@ -15,7 +15,7 @@
result_dict = {}
model = context["model"]
- session = context["session"]
+ session = model.Session
if isinstance(obj, sqlalchemy.engine.base.RowProxy):
fields = obj.keys()
--- a/ckan/logic/action/get.py Wed Jun 15 10:08:24 2011 +0100
+++ b/ckan/logic/action/get.py Wed Jun 15 13:32:43 2011 +0100
@@ -1,13 +1,16 @@
+from sqlalchemy.sql import select
from ckan.logic import NotFound, check_access
from ckan.plugins import (PluginImplementations,
IGroupController,
IPackageController)
import ckan.authz
+from ckan.lib.dictization import table_dictize
from ckan.lib.dictization.model_dictize import group_to_api1, group_to_api2
from ckan.lib.dictization.model_dictize import (package_to_api1,
package_to_api2,
package_dictize,
+ resource_list_dictize,
group_dictize)
@@ -21,6 +24,34 @@
packages = query.all()
return [getattr(p, ref_package_by) for p in packages]
+def current_package_list_with_resources(context):
+ model = context["model"]
+ user = context["user"]
+ limit = context.get("limit")
+
+ q = ckan.authz.Authorizer().authorized_query(user, model.PackageRevision)
+ if limit:
+ q = q.limit(limit)
+ pack_rev = q.all()
+ package_list = []
+ for package in pack_rev:
+ result_dict = table_dictize(package, context)
+ res_rev = model.resource_revision_table
+ resource_group = model.resource_group_table
+ q = select([res_rev], from_obj = res_rev.join(resource_group,
+ resource_group.c.id == res_rev.c.resource_group_id))
+ q = q.where(resource_group.c.package_id == package.id)
+ result = q.where(res_rev.c.current == True).execute()
+ result_dict["resources"] = resource_list_dictize(result, context)
+ license_id = result_dict['license_id']
+ if license_id:
+ isopen = model.Package.get_license_register()[license_id].isopen()
+ result_dict['isopen'] = isopen
+ else:
+ result_dict['isopen'] = False
+ package_list.append(result_dict)
+ return package_list
+
def revision_list(context):
model = context["model"]
--- a/ckan/templates/_util.html Wed Jun 15 10:08:24 2011 +0100
+++ b/ckan/templates/_util.html Wed Jun 15 13:32:43 2011 +0100
@@ -91,6 +91,56 @@
</li></ul>
+ <ul py:def="package_list_from_dict(packages)" class="packages">
+ <li py:for="package in packages"
+ class="${'fullyopen' if (package.isopen and package.get('resources')) else None}">
+ <div class="header">
+ <span class="title">
+ ${h.link_to(package.get('title') or package.get('name'), h.url_for(controller='package', action='read', id=package.get('name')))}
+ </span>
+
+ <div class="search_meta">
+ <py:if test="package.resources">
+ <ul class="package_formats">
+ <py:for each="resource in package.resources">
+ <py:if test="resource.get('format')">
+ <li>${resource.get('format')}</li>
+ </py:if>
+ </py:for>
+ </ul>
+ </py:if>
+ <ul class="openness">
+ <py:if test="package.isopen">
+ <li>
+ <a href="http://opendefinition.org/okd/" title="This package satisfies the Open Definition.">
+ <img src="http://assets.okfn.org/images/ok_buttons/od_80x15_blue.png" alt="[Open Data]" />
+ </a>
+ </li>
+ <li>
+ <a href="http://opendefinition.org/okd/" title="This package satisfies the Open Definition.">
+ <img src="http://assets.okfn.org/images/ok_buttons/oc_80x15_blue.png" alt="[Open Content]" />
+ </a>
+ </li>
+ </py:if>
+ <py:if test="not package.isopen">
+ <li>
+ <span class="closed">
+ ${h.icon('lock')} Not Openly Licensed
+ </span>
+ </li>
+ </py:if>
+ </ul>
+ </div>
+ </div>
+ <div class="extract">
+ ${h.markdown_extract(package.notes)}
+ </div>
+ <!--ul py:if="package.tags" class="tags">
+ <li py:for="tag in package.tags">${tag.name}</li>
+ </ul-->
+ </li>
+ </ul>
+
<!--! List of data package groups: pass in a collection of data package groups
and this renders the standard group listing --><table py:def="group_list(groups)" class="groups">
--- a/ckan/templates/home/index.html Wed Jun 15 10:08:24 2011 +0100
+++ b/ckan/templates/home/index.html Wed Jun 15 13:32:43 2011 +0100
@@ -64,7 +64,7 @@
</py:if><h4>Recently changed packages</h4>
- ${package_list(c.latest_packages)}
+ ${package_list_from_dict(c.latest_packages)}
<p><a href="${h.url_for(controller='revision', action='index',
id=None)}">View revision log »</a></p>
http://bitbucket.org/okfn/ckan/changeset/dcc37830198c/
changeset: dcc37830198c
branch: feature-1141-moderated-edits-ajax
user: kindly
date: 2011-06-15 21:41:39
summary: [moderated edits] do not override deleted revisions
affected #: 1 file (18 bytes)
--- a/ckan/migration/versions/039_add_expired_id_and_dates.py Wed Jun 15 13:32:43 2011 +0100
+++ b/ckan/migration/versions/039_add_expired_id_and_dates.py Wed Jun 15 20:41:39 2011 +0100
@@ -201,7 +201,7 @@
-- change state of revision tables
-update revision set state = 'active', approved_timestamp = timestamp;
+update revision set approved_timestamp = timestamp;
'''
migrate_engine.execute('begin; ' + make_missing_revisions + update_schema + ' commit;')
http://bitbucket.org/okfn/ckan/changeset/96d91479e096/
changeset: 96d91479e096
branch: release-v1.4.1
user: kindly
date: 2011-06-16 13:02:02
summary: [model] fix bug so you can delete packages with groups
affected #: 8 files (1015 bytes)
--- a/ckan/controllers/group.py Sun Jun 12 20:18:17 2011 +0100
+++ b/ckan/controllers/group.py Thu Jun 16 12:02:02 2011 +0100
@@ -52,7 +52,6 @@
query = authz.Authorizer().authorized_query(c.user, model.Group)
query = query.order_by(model.Group.name.asc())
query = query.order_by(model.Group.title.asc())
- query = query.options(eagerload_all('packages'))
c.page = Page(
collection=query,
page=request.params.get('page', 1),
--- a/ckan/lib/search/sql.py Sun Jun 12 20:18:17 2011 +0100
+++ b/ckan/lib/search/sql.py Thu Jun 16 12:02:02 2011 +0100
@@ -192,7 +192,7 @@
group = model.Group.by_name(unicode(term), autoflush=False)
if group:
# need to keep joining for each filter
- q = q.join('groups', aliased=True).filter(
+ q = q.join('package_group_all', 'group', aliased=True).filter(
model.Group.id==group.id)
else:
# unknown group, so torpedo search
--- a/ckan/lib/stats.py Sun Jun 12 20:18:17 2011 +0100
+++ b/ckan/lib/stats.py Thu Jun 16 12:02:02 2011 +0100
@@ -49,6 +49,7 @@
package_group = table('package_group')
s = select([package_group.c.group_id, func.count(package_group.c.package_id)]).\
group_by(package_group.c.group_id).\
+ where(package_group.c.group_id!=None).\
order_by(func.count(package_group.c.package_id).desc()).\
limit(limit)
res_ids = model.Session.execute(s).fetchall()
--- a/ckan/model/group.py Sun Jun 12 20:18:17 2011 +0100
+++ b/ckan/model/group.py Thu Jun 16 12:02:02 2011 +0100
@@ -8,6 +8,7 @@
from types import make_uuid
import vdm.sqlalchemy
from ckan.model import extension
+from sqlalchemy.ext.associationproxy import association_proxy
__all__ = ['group_table', 'Group', 'package_revision_table',
'PackageGroup', 'GroupRevision', 'PackageGroupRevision']
@@ -57,7 +58,6 @@
def get(cls, reference):
'''Returns a group object referenced by its id or name.'''
query = Session.query(cls).filter(cls.id==reference)
- query = query.options(eagerload_all('packages'))
group = query.first()
if group == None:
group = cls.by_name(reference)
@@ -67,7 +67,7 @@
def active_packages(self, load_eager=True):
query = Session.query(Package).\
filter_by(state=vdm.sqlalchemy.State.ACTIVE).\
- join('groups').filter_by(id=self.id)
+ join('package_group_all', 'group').filter_by(id=self.id)
if load_eager:
query = query.options(eagerload_all('package_tags.tag'))
query = query.options(eagerload_all('resource_groups_all.resources_all'))
@@ -119,12 +119,8 @@
return '<Group %s>' % self.name
-mapper(Group, group_table, properties={
- 'packages': relation(Package, secondary=package_group_table,
- backref='groups',
- order_by=package_table.c.name
- )},
- extension=[vdm.sqlalchemy.Revisioner(group_revision_table),],
+mapper(Group, group_table,
+ extension=[vdm.sqlalchemy.Revisioner(group_revision_table),],
)
@@ -132,11 +128,26 @@
GroupRevision = vdm.sqlalchemy.create_object_version(mapper, Group,
group_revision_table)
-
-mapper(PackageGroup, package_group_table,
+mapper(PackageGroup, package_group_table, properties={
+ 'group': relation(Group,
+ backref=backref('package_group_all', cascade='all, delete-orphan'),
+ ),
+ 'package': relation(Package,
+ backref=backref('package_group_all', cascade='all, delete-orphan'),
+ ),
+},
extension=[vdm.sqlalchemy.Revisioner(package_group_revision_table),],
)
+def _create_group(group):
+ return PackageGroup(group=group)
+
+def _create_package(package):
+ return PackageGroup(package=package)
+
+Package.groups = association_proxy('package_group_all', 'group', creator=_create_group)
+Group.packages = association_proxy('package_group_all', 'package', creator=_create_package)
+
vdm.sqlalchemy.modify_base_object_mapper(PackageGroup, Revision, State)
PackageGroupRevision = vdm.sqlalchemy.create_object_version(mapper, PackageGroup,
--- a/ckan/model/package.py Sun Jun 12 20:18:17 2011 +0100
+++ b/ckan/model/package.py Thu Jun 16 12:02:02 2011 +0100
@@ -528,4 +528,3 @@
return fields
-
--- a/ckan/tests/lib/test_dictization_schema.py Sun Jun 12 20:18:17 2011 +0100
+++ b/ckan/tests/lib/test_dictization_schema.py Thu Jun 16 12:02:02 2011 +0100
@@ -132,14 +132,15 @@
data = group_dictize(group, context)
converted_data, errors = validate(data, default_group_schema(), context)
- group.packages.sort(key=lambda x:x.id)
+ group_pack = sorted(group.packages, key=lambda x:x.id)
+
converted_data["packages"] = sorted(converted_data["packages"], key=lambda x:x["id"])
expected = {'description': u'These are books that David likes.',
'id': group.id,
'name': u'david',
- 'packages': sorted([{'id': group.packages[0].id},
- {'id': group.packages[1].id,
+ 'packages': sorted([{'id': group_pack[0].id},
+ {'id': group_pack[1].id,
}], key=lambda x:x["id"]),
'title': u"Dave's books"}
--- a/ckan/tests/misc/test_stats.py Sun Jun 12 20:18:17 2011 +0100
+++ b/ckan/tests/misc/test_stats.py Thu Jun 16 12:02:02 2011 +0100
@@ -15,6 +15,8 @@
@classmethod
def setup_class(self):
+ from ckan import plugins
+ plugins.load('synchronous_search')
CreateTestData.create_search_test_data()
@classmethod
@@ -186,7 +188,7 @@
model.repo.commit_and_remove()
return pkg_name
-class TestRateStatsSimple(TimedRevision):
+class fdsTestRateStatsSimple(TimedRevision):
@classmethod
def setup_class(self):
model.repo.init_db()
@@ -245,7 +247,7 @@
assert res == 1, res
-class TestRateStats(TimedRevision):
+class fdsTestRateStats(TimedRevision):
@classmethod
def setup_class(self):
model.repo.init_db()
--- a/ckan/tests/models/test_package.py Sun Jun 12 20:18:17 2011 +0100
+++ b/ckan/tests/models/test_package.py Thu Jun 16 12:02:02 2011 +0100
@@ -26,6 +26,7 @@
@classmethod
def teardown_class(self):
pkg1 = model.Session.query(model.Package).filter_by(name=self.name).one()
+
pkg1.purge()
model.Session.commit()
model.repo.rebuild_db()
@@ -373,3 +374,17 @@
test_res(diff, self.res1, 'hash', 'abc123')
test_res(diff, self.res1, 'state', 'active')
test_res(diff, self.res2, 'url', 'http://url2.com')
+
+class TestPackagePurge:
+ @classmethod
+ def setup_class(self):
+ CreateTestData.create()
+ def test_purge(self):
+ pkgs = model.Session.query(model.Package).all()
+ for p in pkgs:
+ p.purge()
+ model.Session.commit()
+ pkgs = model.Session.query(model.Package).all()
+ assert len(pkgs) == 0
+
+
http://bitbucket.org/okfn/ckan/changeset/8963309b4158/
changeset: 8963309b4158
branch: release-v1.4.1
user: kindly
date: 2011-06-16 13:03:09
summary: [model] merge 1.4.1
affected #: 5 files (2.5 KB)
--- a/ckan/controllers/group.py Thu Jun 16 12:02:02 2011 +0100
+++ b/ckan/controllers/group.py Thu Jun 16 12:03:09 2011 +0100
@@ -71,7 +71,11 @@
import ckan.misc
format = ckan.misc.MarkdownFormat()
desc_formatted = format.to_html(c.group.description)
- desc_formatted = genshi.HTML(desc_formatted)
+ try:
+ desc_formatted = genshi.HTML(desc_formatted)
+ except genshi.ParseError, e:
+ log.error('Could not print group description: %r Error: %r', c.group.description, e)
+ desc_formatted = 'Error: Could not parse group description'
c.group_description_formatted = desc_formatted
c.group_admins = self.authorizer.get_admins(c.group)
--- a/ckan/controllers/user.py Thu Jun 16 12:02:02 2011 +0100
+++ b/ckan/controllers/user.py Thu Jun 16 12:03:09 2011 +0100
@@ -1,4 +1,5 @@
import re
+import logging
import genshi
from sqlalchemy import or_, func, desc
@@ -6,6 +7,8 @@
import ckan.misc
from ckan.lib.base import *
+log = logging.getLogger(__name__)
+
def login_form():
return render('user/login_form.html').replace('FORM_ACTION', '%s')
@@ -138,10 +141,15 @@
c.user_email = request.params.getone('email')
elif 'save' in request.params:
try:
- rev = model.repo.new_revision()
- rev.author = c.author
- rev.message = _(u'Changed user details')
- user.about = request.params.getone('about')
+ about = request.params.getone('about')
+ if 'http://' in about or 'https://' in about:
+ msg = _('Edit not allowed as it looks like spam. Please avoid links in your description.')
+ h.flash_error(msg)
+ c.user_about = about
+ c.user_fullname = request.params.getone('fullname')
+ c.user_email = request.params.getone('email')
+ return render('user/edit.html')
+ user.about = about
user.fullname = request.params.getone('fullname')
user.email = request.params.getone('email')
try:
@@ -164,8 +172,13 @@
def _format_about(self, about):
about_formatted = ckan.misc.MarkdownFormat().to_html(about)
- return genshi.HTML(about_formatted)
-
+ try:
+ html = genshi.HTML(about_formatted)
+ except genshi.ParseError, e:
+ log.error('Could not print "about" field Field: %r Error: %r', about, e)
+ html = 'Error: Could not parse About text'
+ return html
+
def _get_form_password(self):
password1 = request.params.getone('password1')
password2 = request.params.getone('password2')
--- a/ckan/public/css/forms.css Thu Jun 16 12:02:02 2011 +0100
+++ b/ckan/public/css/forms.css Thu Jun 16 12:03:09 2011 +0100
@@ -46,7 +46,7 @@
input.title {
font-size: 1.5em; }
input.short {
- width: 15em; }
+ width: 10em; }
input.medium-width {
width: 25em; }
input.long {
--- a/ckan/tests/functional/test_package.py Thu Jun 16 12:02:02 2011 +0100
+++ b/ckan/tests/functional/test_package.py Thu Jun 16 12:03:09 2011 +0100
@@ -870,7 +870,7 @@
assert field_name in res
fv = res.forms['package-edit']
fv[prefix + 'groups__0__id'] = grp.id
- res = fv.submit('preview',extra_environ={'REMOTE_USER':'russianfan'})
+ res = fv.submit('preview', extra_environ={'REMOTE_USER':'russianfan'})
assert not 'error' in res
res = fv.submit('save', extra_environ={'REMOTE_USER':'russianfan'})
res = res.follow()
--- a/ckan/tests/functional/test_user.py Thu Jun 16 12:02:02 2011 +0100
+++ b/ckan/tests/functional/test_user.py Thu Jun 16 12:03:09 2011 +0100
@@ -21,6 +21,7 @@
CreateTestData.create_user('unfinisher', about='<a href="http://unfinished.tag')
CreateTestData.create_user('uncloser', about='<a href="http://unclosed.tag">')
CreateTestData.create_user('spammer', about=u'<a href="http://mysite">mysite</a><a href=\u201dhttp://test2\u201d>test2</a>')
+ CreateTestData.create_user('spammer2', about=u'<a href="http://spamsite1.com\u201d>spamsite1</a>\r\n<a href="http://www.spamsite2.com\u201d>spamsite2</a>\r\n')
@classmethod
def teardown_class(self):
@@ -98,6 +99,15 @@
'href="TAG MALFORMED"',
'target="_blank"',
'rel="nofollow"')
+
+ def test_user_read_about_spam2(self):
+ user = model.User.by_name(u'spammer2')
+ offset = '/user/%s' % user.id
+ res = self.app.get(offset, status=200)
+ main_res = self.main_div(res)
+ assert 'spammer2' in res, res
+ assert 'spamsite2' not in res, res
+ assert 'Error: Could not parse About text' in res, res
def test_user_login(self):
offset = url_for(controller='user', action='login', id=None)
@@ -206,6 +216,32 @@
main_res = self.main_div(res)
assert new_about in main_res, main_res
+ def test_edit_spammer(self):
+ # create user
+ username = 'testeditspam'
+ about = u'Test About <a href="http://spamsite.net">spamsite</a>'
+ user = model.User.by_name(unicode(username))
+ if not user:
+ model.Session.add(model.User(name=unicode(username), about=about,
+ password='letmein'))
+ model.repo.commit_and_remove()
+ user = model.User.by_name(unicode(username))
+
+ # edit
+ offset = url_for(controller='user', action='edit', id=user.id)
+ res = self.app.get(offset, status=200, extra_environ={'REMOTE_USER':username})
+ main_res = self.main_div(res)
+ assert 'Edit User: ' in main_res, main_res
+ assert 'Test About <a href="http://spamsite.net">spamsite</a>' in main_res, main_res
+ fv = res.forms['user-edit']
+ res = fv.submit('preview', extra_environ={'REMOTE_USER':username})
+ # commit
+ res = fv.submit('save', extra_environ={'REMOTE_USER':username})
+ assert res.status == 200, res.status
+ main_res = self.main_div(res)
+ assert 'looks like spam' in main_res, main_res
+ assert 'Edit User: ' in main_res, main_res
+
############
# Disabled
Repository URL: https://bitbucket.org/okfn/ckan/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list