[ckan-changes] commit/ckan: 2 new changesets
Bitbucket
commits-noreply at bitbucket.org
Thu Jun 23 11:34:27 UTC 2011
2 new changesets in ckan:
http://bitbucket.org/okfn/ckan/changeset/8a773139385b/
changeset: 8a773139385b
branch: feature-1141-moderated-edits-ajax
user: dread
date: 2011-06-23 12:46:33
summary: [migration]: Removed user group migration as it is not related to moderated edits. It was only added here because of (now resolved) branching problems.
affected #: 1 file (0 bytes)
--- a/ckan/migration/versions/040_add_user_group_tables.py Wed Jun 22 23:06:19 2011 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,67 +0,0 @@
-from migrate import *
-
-def upgrade(migrate_engine):
-
- migrate_engine.execute('''
-
-BEGIN;
-
-CREATE TABLE user_group (
- id text NOT NULL,
- name text NOT NULL,
- parent_id text
-);
-
-CREATE TABLE user_group_extra (
- id text NOT NULL,
- user_group_id text NOT NULL,
- "key" text NOT NULL,
- "value" text NOT NULL
-);
-
-CREATE TABLE user_group_package (
- id text NOT NULL,
- user_group_id text NOT NULL,
- package_id text NOT NULL,
- capacity text
-);
-
-CREATE TABLE user_group_user (
- id text NOT NULL,
- user_group_id text NOT NULL,
- user_id text NOT NULL,
- capacity text
-);
-
-
-ALTER TABLE user_group
- ADD CONSTRAINT user_group_pkey PRIMARY KEY (id);
-
-ALTER TABLE user_group_extra
- ADD CONSTRAINT user_group_extra_pkey PRIMARY KEY (id);
-
-ALTER TABLE user_group_package
- ADD CONSTRAINT user_group_package_pkey PRIMARY KEY (id);
-
-ALTER TABLE user_group_user
- ADD CONSTRAINT user_group_user_pkey PRIMARY KEY (id);
-
-
-
-ALTER TABLE user_group_extra
- ADD CONSTRAINT user_group_extra_user_group_id_fkey FOREIGN KEY (user_group_id) REFERENCES user_group(id);
-
-ALTER TABLE user_group_package
- ADD CONSTRAINT user_group_package_package_id_fkey FOREIGN KEY (package_id) REFERENCES package(id);
-
-ALTER TABLE user_group_package
- ADD CONSTRAINT user_group_package_user_group_id_fkey FOREIGN KEY (user_group_id) REFERENCES user_group(id);
-
-ALTER TABLE user_group_user
- ADD CONSTRAINT user_group_user_user_group_id_fkey FOREIGN KEY (user_group_id) REFERENCES user_group(id);
-
-ALTER TABLE user_group_user
- ADD CONSTRAINT user_group_user_user_id_fkey FOREIGN KEY (user_id) REFERENCES "user"(id);
-
-COMMIT;
-''')
http://bitbucket.org/okfn/ckan/changeset/79785e105de5/
changeset: 79785e105de5
user: dread
date: 2011-06-23 13:30:51
summary: [merge] from feature-1141-moderated-edits-ajax.
affected #: 44 files (54.6 KB)
--- a/CHANGELOG.txt Thu Jun 23 11:19:31 2011 +0100
+++ b/CHANGELOG.txt Thu Jun 23 12:30:51 2011 +0100
@@ -1,6 +1,13 @@
CKAN CHANGELOG
++++++++++++++
+v1.4.2 2011-XX-XX
+=================
+Major:
+ * Packages revisions can be marked as 'moderated' (#1141)
+ * Viewing of a package at any revision (#1141)
+
+
v1.4.1 2011-XX-XX
=================
Minor:
@@ -13,6 +20,7 @@
Bug fixes
* Duplicate authorization roles were difficult to delete (#1083)
+
v1.4 2011-05-19
===============
Major:
--- a/ckan/authz.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/authz.py Thu Jun 23 12:30:51 2011 +0100
@@ -73,7 +73,7 @@
# check it's active
if domain_object.__class__ != type and hasattr(domain_object, 'state'):
- if domain_object.state != model.State.ACTIVE:
+ if domain_object.state == model.State.DELETED:
return False
# check if any of the roles allows the action requested
@@ -184,7 +184,6 @@
user = model.User.by_name(username, autoflush=False)
else:
user = None
- entity.roles.property.mapper.class_
visitor = model.User.by_name(model.PSEUDO_USER__VISITOR, autoflush=False)
logged_in = model.User.by_name(model.PSEUDO_USER__LOGGED_IN,
autoflush=False)
@@ -193,8 +192,15 @@
# need to use this in the queries below as if we use
# model.UserObjectRole a cross join happens always
# returning all the roles.
- role_cls = entity.roles.property.mapper.class_
- q = q.outerjoin('roles')
+ if hasattr(entity, 'continuity'):
+ q = q.filter_by(current=True)
+ q = q.outerjoin('continuity', 'roles')
+ continuity = entity.continuity.property.mapper.class_
+ role_cls = continuity.roles.property.mapper.class_
+ else:
+ role_cls = entity.roles.property.mapper.class_
+ q = q.outerjoin('roles')
+
if hasattr(entity, 'state'):
state = entity.state
else:
@@ -209,13 +215,13 @@
q = q.filter(sa.or_(
sa.and_(role_cls.role==model.RoleAction.role,
model.RoleAction.action==action,
- state and state==model.State.ACTIVE),
+ state and state!=model.State.DELETED),
role_cls.role==model.Role.ADMIN))
else:
q = q.filter(
sa.and_(role_cls.role==model.RoleAction.role,
model.RoleAction.action==action,
- state and state==model.State.ACTIVE),
+ state and state!=model.State.DELETED),
)
q = q.filter(sa.or_(*filters))
q = q.distinct()
--- a/ckan/config/routing.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/config/routing.py Thu Jun 23 12:30:51 2011 +0100
@@ -179,11 +179,14 @@
]))
)
map.connect('/package', controller='package', action='index')
+ map.connect('/package/{action}/{id}/{revision}', controller='package', action='read_ajax')
map.connect('/package/{action}/{id}', controller='package',
requirements=dict(action='|'.join([
'edit',
'authz',
- 'history'
+ 'history',
+ 'read_ajax',
+ 'history_ajax',
]))
)
map.connect('/package/{id}', controller='package', action='read')
--- a/ckan/controllers/home.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/controllers/home.py Thu Jun 23 12:30:51 2011 +0100
@@ -5,6 +5,7 @@
from genshi.template import NewTextTemplate
from ckan.authz import Authorizer
+from ckan.logic.action.get import current_package_list_with_resources
from ckan.i18n import set_session_locale
from ckan.lib.search import query_for, QueryOptions, SearchError
from ckan.lib.cache import proxy_cache, get_cache_expires
@@ -46,10 +47,9 @@
c.facets = query.facets
c.fields = []
c.package_count = query.count
- c.latest_packages = self.authorizer.authorized_query(c.user, model.Package)\
- .join('revision').order_by(model.Revision.timestamp.desc())\
- .limit(5).all()
-
+ c.latest_packages = current_package_list_with_resources({'model': model,
+ 'user': c.user,
+ 'limit': 5})
return render('home/index.html', cache_key=cache_key,
cache_expire=cache_expires)
--- a/ckan/controllers/package.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/controllers/package.py Thu Jun 23 12:30:51 2011 +0100
@@ -1,6 +1,9 @@
import logging
import urlparse
from urllib import urlencode
+import json
+import datetime
+import re
from sqlalchemy.orm import eagerload_all
from sqlalchemy import or_
@@ -8,6 +11,7 @@
from pylons import config, cache
from pylons.i18n import get_lang, _
from autoneg.accept import negotiate
+from babel.dates import format_date, format_datetime, format_time
import ckan.logic.action.create as create
import ckan.logic.action.update as update
@@ -21,8 +25,9 @@
from ckan.lib.package_saver import PackageSaver, ValidationException
from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
from ckan.logic import NotFound, NotAuthorized, ValidationError
-from ckan.logic import tuplize_dict, clean_dict, parse_params
+from ckan.logic import tuplize_dict, clean_dict, parse_params, flatten_to_string_key
from ckan.plugins import PluginImplementations, IPackageController
+from ckan.lib.dictization import table_dictize
import ckan.forms
import ckan.authz
import ckan.rating
@@ -171,11 +176,26 @@
@proxy_cache()
def read(self, id):
-
+ context = {'model': model, 'session': model.Session,
+ 'user': c.user or c.author, 'extras_as_string': True,
+ 'schema': self._form_to_db_schema(),
+ 'id': id}
+ split = id.split('@')
+ if len(split) == 2:
+ context['id'], revision = split
+ try:
+ date = datetime.datetime(*map(int, re.split('[^\d]', revision)))
+ context['revision_date'] = date
+ except ValueError:
+ context['revision_id'] = revision
#check if package exists
- c.pkg = model.Package.get(id)
- if c.pkg is None:
+ try:
+ c.pkg_dict = get.package_show(context)
+ c.pkg = context['package']
+ except NotFound:
abort(404, _('Package not found'))
+ except NotAuthorized:
+ abort(401, _('Unauthorized to read package %s') % id)
cache_key = self._pkg_cache_key(c.pkg)
etag_cache(cache_key)
@@ -194,38 +214,33 @@
rdf_url = '%s%s.%s' % (config['rdf_packages'], c.pkg.id, exts[0])
redirect(rdf_url, code=303)
break
-
- #is the user allowed to see this package?
- auth_for_read = self.authorizer.am_authorized(c, model.Action.READ, c.pkg)
- if not auth_for_read:
- abort(401, _('Unauthorized to read package %s') % id)
-
- #render the package
- PackageSaver().render_package(c.pkg)
- for item in self.extensions:
- item.read(c.pkg)
+
+ PackageSaver().render_package(c.pkg_dict, context)
return render('package/read.html')
def comments(self, id):
+ context = {'model': model, 'session': model.Session,
+ 'user': c.user or c.author, 'extras_as_string': True,
+ 'schema': self._form_to_db_schema(),
+ 'id': id}
#check if package exists
- c.pkg = model.Package.get(id)
- if c.pkg is None:
+ try:
+ c.pkg_dict = get.package_show(context)
+ c.pkg = context['package']
+ except NotFound:
abort(404, _('Package not found'))
+ except NotAuthorized:
+ abort(401, _('Unauthorized to read package %s') % id)
# used by disqus plugin
c.current_package_id = c.pkg.id
- #is the user allowed to see this package?
- auth_for_read = self.authorizer.am_authorized(c, model.Action.READ, c.pkg)
- if not auth_for_read:
- abort(401, _('Unauthorized to read package %s') % id)
-
for item in self.extensions:
item.read(c.pkg)
#render the package
- PackageSaver().render_package(c.pkg)
+ PackageSaver().render_package(c.pkg_dict)
return render('package/comments.html')
@@ -318,7 +333,8 @@
'user': c.user or c.author, 'extras_as_string': True,
'preview': 'preview' in request.params,
'save': 'save' in request.params,
- 'id': id,
+ 'id': id, 'moderated': config.get('moderated'),
+ 'pending': True,
'schema': self._form_to_db_schema()}
if (context['save'] or context['preview']) and not data:
@@ -347,6 +363,53 @@
c.form = render(self.package_form, extra_vars=vars)
return render('package/edit.html')
+ def read_ajax(self, id, revision=None):
+ context = {'model': model, 'session': model.Session,
+ 'user': c.user or c.author,
+ 'id': id, 'extras_as_string': True,
+ 'schema': self._form_to_db_schema(),
+ 'revision_id': revision}
+
+ try:
+ data = get.package_show(context)
+ schema = self._db_to_form_schema()
+ if schema:
+ data, errors = validate(data, schema)
+ except NotAuthorized:
+ abort(401, _('Unauthorized to read package %s') % '')
+
+ ## hack as db_to_form schema should have this
+ data['tag_string'] = ' '.join([tag['name'] for tag in data.get('tags', [])])
+ data.pop('tags')
+ data = flatten_to_string_key(data)
+ response.headers['Content-Type'] = 'application/json;charset=utf-8'
+ return json.dumps(data)
+
+ def history_ajax(self, id):
+
+ context = {'model': model, 'session': model.Session,
+ 'user': c.user or c.author,
+ 'id': id, 'extras_as_string': True}
+ pkg = model.Package.get(id)
+ data = []
+ approved = False
+ for num, (revision, revision_obj) in enumerate(pkg.all_related_revisions):
+ if not approved and revision.approved_timestamp:
+ current_approved, approved = True, True
+ else:
+ current_approved = False
+
+ data.append({'revision_id': revision.id,
+ 'message': revision.message,
+ 'timestamp': format_datetime(revision.timestamp,
+ locale=(get_lang() or ['en'])[0]),
+ 'author': revision.author,
+ 'approved': bool(revision.approved_timestamp),
+ 'current_approved': current_approved})
+
+ response.headers['Content-Type'] = 'application/json;charset=utf-8'
+ return json.dumps(data)
+
def _save_new(self, context):
try:
data_dict = clean_dict(unflatten(
@@ -356,7 +419,9 @@
pkg = create.package_create(data_dict, context)
if context['preview']:
- PackageSaver().render_package(context['package'])
+ PackageSaver().render_package(pkg, context)
+ c.pkg = context['package']
+ c.pkg_dict = data_dict
c.is_preview = True
c.preview = render('package/read_core.html')
return self.new(data_dict)
@@ -379,12 +444,17 @@
tuplize_dict(parse_params(request.POST))))
self._check_data_dict(data_dict)
context['message'] = data_dict.get('log_message', '')
+ if not context['moderated']:
+ context['pending'] = False
pkg = update.package_update(data_dict, context)
+ if request.params.get('save', '') == 'Approve':
+ update.make_latest_pending_package_active(context)
c.pkg = context['package']
+ c.pkg_dict = pkg
if context['preview']:
c.is_preview = True
- PackageSaver().render_package(context['package'])
+ PackageSaver().render_package(pkg, context)
c.preview = render('package/read_core.html')
return self.edit(id, data_dict)
--- a/ckan/lib/create_test_data.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/lib/create_test_data.py Thu Jun 23 12:30:51 2011 +0100
@@ -370,10 +370,12 @@
description=u'Roger likes these books.')
for obj in [david, roger]:
model.Session.add(obj)
+
cls.group_names.add(u'david')
cls.group_names.add(u'roger')
- david.packages = [pkg1, pkg2]
- roger.packages = [pkg1]
+ model.Session.add(model.PackageGroup(package=pkg1, group=david))
+ model.Session.add(model.PackageGroup(package=pkg2, group=david))
+ model.Session.add(model.PackageGroup(package=pkg1, group=roger))
# authz
model.Session.add_all([
model.User(name=u'tester', apikey=u'tester', password=u'tester'),
--- a/ckan/lib/dictization/__init__.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/lib/dictization/__init__.py Thu Jun 23 12:30:51 2011 +0100
@@ -15,15 +15,21 @@
result_dict = {}
model = context["model"]
- session = context["session"]
+ session = model.Session
- ModelClass = obj.__class__
- table = class_mapper(ModelClass).mapped_table
-
- fields = [field.name for field in table.c]
+ if isinstance(obj, sqlalchemy.engine.base.RowProxy):
+ fields = obj.keys()
+ else:
+ ModelClass = obj.__class__
+ table = class_mapper(ModelClass).mapped_table
+ fields = [field.name for field in table.c]
for field in fields:
name = field
+ if name in ('current', 'expired_timestamp', 'expired_id'):
+ continue
+ if name == 'continuity_id':
+ continue
value = getattr(obj, name)
if value is None:
result_dict[name] = value
@@ -45,8 +51,11 @@
'''Get a list of model object and represent it as a list of dicts'''
result_list = []
+ active = context.get('active', True)
for obj in obj_list:
+ if active and obj.state not in ('active', 'pending'):
+ continue
result_list.append(table_dictize(obj, context))
return sorted(result_list, key=sort_key)
@@ -111,6 +120,10 @@
continue
setattr(obj, key, value)
+ if context.get('pending'):
+ if session.is_modified(obj, include_collections=False):
+ obj.state = 'pending'
+
session.add(obj)
return obj
--- a/ckan/lib/dictization/model_dictize.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/lib/dictization/model_dictize.py Thu Jun 23 12:30:51 2011 +0100
@@ -1,8 +1,11 @@
from pylons import config
+from sqlalchemy.sql import select, and_
+import datetime
from ckan.lib.dictization import (obj_list_dictize,
obj_dict_dictize,
table_dictize)
+from ckan.logic import NotFound
import ckan.misc
import json
@@ -10,18 +13,26 @@
def group_list_dictize(obj_list, context, sort_key=lambda x:x):
+ active = context.get('active', True)
+
result_list = []
for obj in obj_list:
group_dict = table_dictize(obj, context)
group_dict.pop('created')
+ if active and obj.state not in ('active', 'pending'):
+ continue
+
result_list.append(group_dict)
return sorted(result_list, key=sort_key)
def resource_list_dictize(res_list, context):
+ active = context.get('active', True)
result_list = []
for res in res_list:
+ if active and res.state not in ('active', 'pending'):
+ continue
result_list.append(resource_dictize(res, context))
return sorted(result_list, key=lambda x: x["position"])
@@ -41,6 +52,20 @@
return sorted(result_list, key=lambda x: x["key"])
+def extras_list_dictize(extras_list, context):
+ result_list = []
+ active = context.get('active', True)
+ for extra in extras_list:
+ if active and extra.state not in ('active', 'pending'):
+ continue
+ dictized = table_dictize(extra, context)
+ value = dictized["value"]
+ if not(context.get("extras_as_string") and isinstance(value, basestring)):
+ dictized["value"] = json.dumps(value)
+ result_list.append(dictized)
+
+ return sorted(result_list, key=lambda x: x["key"])
+
def resource_dictize(res, context):
resource = table_dictize(res, context)
extras = resource.pop("extras", None)
@@ -48,23 +73,76 @@
resource.update(extras)
return resource
+def _execute_with_revision(q, rev_table, context):
+
+ model = context['model']
+ meta = model.meta
+ session = model.Session
+ revision_id = context.get('revision_id')
+ revision_date = context.get('revision_date')
+ pending = context.get('pending')
+
+ if revision_id:
+ revision_date = session.query(context['model'].Revision).filter_by(
+ id=revision_id).one().timestamp
+
+ if revision_date:
+ q = q.where(rev_table.c.revision_timestamp <= revision_date)
+ q = q.where(rev_table.c.expired_timestamp > revision_date)
+ elif pending:
+ q = q.where(rev_table.c.expired_timestamp == datetime.datetime(9999, 12, 31))
+ else:
+ q = q.where(rev_table.c.current == True)
+
+ return session.execute(q)
+
+
def package_dictize(pkg, context):
-
- result_dict = table_dictize(pkg, context)
-
- result_dict["resources"] = resource_list_dictize(pkg.resources, context)
-
- result_dict["tags"] = obj_list_dictize(
- pkg.tags, context, lambda x: x["name"])
- result_dict["extras"] = extras_dict_dictize(
- pkg._extras, context)
- result_dict["groups"] = group_list_dictize(
- pkg.groups, context, lambda x: x["name"])
- result_dict["relationships_as_subject"] = obj_list_dictize(
- pkg.relationships_as_subject, context)
- result_dict["relationships_as_object"] = obj_list_dictize(
- pkg.relationships_as_object, context)
-
+ model = context['model']
+ #package
+ package_rev = model.package_revision_table
+ q = select([package_rev]).where(package_rev.c.id == pkg.id)
+ result = _execute_with_revision(q, package_rev, context).first()
+ if not result:
+ raise NotFound
+ result_dict = table_dictize(result, context)
+ #resources
+ res_rev = model.resource_revision_table
+ resource_group = model.resource_group_table
+ q = select([res_rev], from_obj = res_rev.join(resource_group,
+ resource_group.c.id == res_rev.c.resource_group_id))
+ q = q.where(resource_group.c.package_id == pkg.id)
+ result = _execute_with_revision(q, res_rev, context)
+ result_dict["resources"] = resource_list_dictize(result, context)
+ #tags
+ tag_rev = model.package_tag_revision_table
+ tag = model.tag_table
+ q = select([tag, tag_rev.c.state, tag_rev.c.revision_timestamp],
+ from_obj=tag_rev.join(tag, tag.c.id == tag_rev.c.tag_id)
+ ).where(tag_rev.c.package_id == pkg.id)
+ result = _execute_with_revision(q, tag_rev, context)
+ result_dict["tags"] = obj_list_dictize(result, context, lambda x: x["name"])
+ #extras
+ extra_rev = model.extra_revision_table
+ q = select([extra_rev]).where(extra_rev.c.package_id == pkg.id)
+ result = _execute_with_revision(q, extra_rev, context)
+ result_dict["extras"] = extras_list_dictize(result, context)
+ #groups
+ group_rev = model.package_group_revision_table
+ group = model.group_table
+ q = select([group],
+ from_obj=group_rev.join(group, group.c.id == group_rev.c.group_id)
+ ).where(group_rev.c.package_id == pkg.id)
+ result = _execute_with_revision(q, group_rev, context)
+ result_dict["groups"] = obj_list_dictize(result, context)
+ #relations
+ rel_rev = model.package_relationship_revision_table
+ q = select([rel_rev]).where(rel_rev.c.subject_package_id == pkg.id)
+ result = _execute_with_revision(q, rel_rev, context)
+ result_dict["relationships_as_subject"] = obj_list_dictize(result, context)
+ q = select([rel_rev]).where(rel_rev.c.object_package_id == pkg.id)
+ result = _execute_with_revision(q, rel_rev, context)
+ result_dict["relationships_as_object"] = obj_list_dictize(result, context)
return result_dict
def group_dictize(group, context):
@@ -102,12 +180,16 @@
def resource_dict_to_api(res_dict, package_id, context):
res_dict.pop("revision_id")
res_dict.pop("state")
+ res_dict.pop("revision_timestamp")
res_dict["package_id"] = package_id
def package_to_api1(pkg, context):
dictized = package_dictize(pkg, context)
+
+ dictized.pop("revision_timestamp")
+
dictized["groups"] = [group["name"] for group in dictized["groups"]]
dictized["tags"] = [tag["name"] for tag in dictized["tags"]]
dictized["extras"] = dict((extra["key"], json.loads(extra["value"]))
@@ -160,7 +242,10 @@
def package_to_api2(pkg, context):
dictized = package_dictize(pkg, context)
+
dictized["groups"] = [group["id"] for group in dictized["groups"]]
+ dictized.pop("revision_timestamp")
+
dictized["tags"] = [tag["name"] for tag in dictized["tags"]]
dictized["extras"] = dict((extra["key"], json.loads(extra["value"]))
for extra in dictized["extras"])
--- a/ckan/lib/dictization/model_save.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/lib/dictization/model_save.py Thu Jun 23 12:30:51 2011 +0100
@@ -21,31 +21,94 @@
table = class_mapper(model.Resource).mapped_table
fields = [field.name for field in table.c]
-
+
for key, value in res_dict.iteritems():
if isinstance(value, list):
continue
- if key == 'extras':
+ if key in ('extras', 'revision_timestamp'):
continue
if key in fields:
setattr(obj, key, value)
else:
obj.extras[key] = value
+ if context.get('pending'):
+ if session.is_modified(obj, include_collections=False):
+ obj.state = 'pending'
+
session.add(obj)
return obj
-def resource_list_save(res_dicts, context):
+def package_resource_list_save(res_dicts, package, context):
+
+ pending = context.get('pending')
+
+ resource_list = package.resource_groups_all[0].resources_all
+ old_list = package.resource_groups_all[0].resources_all[:]
obj_list = []
for res_dict in res_dicts:
obj = resource_dict_save(res_dict, context)
obj_list.append(obj)
- return obj_list
+ resource_list[:] = obj_list
-def extras_save(extras_dicts, context):
+ for resource in set(old_list) - set(obj_list):
+ if pending and resource.state <> 'deleted':
+ resource.state = 'pending-deleted'
+ else:
+ resource.state = 'deleted'
+ resource_list.append(resource)
+ tag_package_tag = dict((package_tag.tag, package_tag)
+ for package_tag in
+ package.package_tag_all)
+
+
+def package_extras_save(extra_dicts, obj, context):
+
+ allow_partial_update = context.get("allow_partial_update", False)
+ if not extra_dicts and allow_partial_update:
+ return
+ model = context["model"]
+ session = context["session"]
+
+ extras_as_string = context.get("extras_as_string", False)
+ extras_list = obj.extras_list
+ old_extras = dict((extra.key, extra) for extra in extras_list)
+
+ new_extras = {}
+ for extra_dict in extra_dicts:
+ if extra_dict.get("deleted"):
+ continue
+ if extras_as_string:
+ new_extras[extra_dict["key"]] = extra_dict["value"]
+ else:
+ new_extras[extra_dict["key"]] = json.loads(extra_dict["value"])
+ #new
+ for key in set(new_extras.keys()) - set(old_extras.keys()):
+ state = 'pending' if context.get('pending') else 'active'
+ extra = model.PackageExtra(state=state, key=key, value=new_extras[key])
+ session.add(extra)
+ extras_list.append(extra)
+ #changed
+ for key in set(new_extras.keys()) & set(old_extras.keys()):
+ extra = old_extras[key]
+ if new_extras[key] == extra.value:
+ continue
+ state = 'pending' if context.get('pending') else 'active'
+ extra.value = new_extras[key]
+ extra.state = state
+ session.add(extra)
+ #deleted
+ for key in set(old_extras.keys()) - set(new_extras.keys()):
+ extra = old_extras[key]
+ if extra.state == 'deleted':
+ continue
+ state = 'pending-deleted' if context.get('pending') else 'delete'
+ extra.state = state
+
+def group_extras_save(extras_dicts, context):
model = context["model"]
session = context["session"]
@@ -62,25 +125,55 @@
return result_dict
+def package_tag_list_save(tag_dicts, package, context):
-def tag_list_save(tag_dicts, context):
+
+ allow_partial_update = context.get("allow_partial_update", False)
+ if not tag_dicts and allow_partial_update:
+ return
model = context["model"]
session = context["session"]
+ pending = context.get('pending')
- tag_list = []
- for table_dict in tag_dicts:
- obj = table_dict_save(table_dict, model.Tag, context)
- tag_list.append(obj)
+ tag_package_tag = dict((package_tag.tag, package_tag)
+ for package_tag in
+ package.package_tag_all)
- return list(set(tag_list))
+ tags = set()
+ for tag_dict in tag_dicts:
+ obj = table_dict_save(tag_dict, model.Tag, context)
+ tags.add(obj)
-def group_list_save(group_dicts, context):
+ for tag in set(tag_package_tag.keys()) - tags:
+ package_tag = tag_package_tag[tag]
+ if pending and package_tag.state <> 'deleted':
+ package_tag.state = 'pending-deleted'
+ else:
+ package_tag.state = 'deleted'
+
+ for tag in tags - set(tag_package_tag.keys()):
+ state = 'pending' if pending else 'active'
+ package_tag_obj = model.PackageTag(package, tag, state)
+ session.add(package_tag_obj)
+ tag_package_tag[tag] = package_tag_obj
+
+ package.package_tag_all[:] = tag_package_tag.values()
+
+def package_group_list_save(group_dicts, package, context):
+
+ allow_partial_update = context.get("allow_partial_update", False)
+ if not group_dicts and allow_partial_update:
+ return
model = context["model"]
session = context["session"]
+ pending = context.get('pending')
- group_list = []
+ group_package_group = dict((package_group.group, package_group)
+ for package_group in
+ package.package_group_all)
+ groups = set()
for group_dict in group_dicts:
id = group_dict.get("id")
name = group_dict.get("name")
@@ -88,23 +181,55 @@
group = session.query(model.Group).get(id)
else:
group = session.query(model.Group).filter_by(name=name).first()
+ groups.add(group)
- group_list.append(group)
+ for group in groups - set(group_package_group.keys()):
+ package_group_obj = model.PackageGroup(package = package,
+ group = group,
+ state = 'active')
+ session.add(package_group_obj)
+ group_package_group[group] = package_group_obj
- return group_list
+ for group in set(group_package_group.keys()) - groups:
+ group_package_group.pop(group)
+ continue
+ ### this is alternate behavioiur below which is correct
+ ### but not compatible with old behaviour
+ package_group = group_package_group[group]
+ if pending and package_group.state <> 'deleted':
+ package_group.state = 'pending-deleted'
+ else:
+ package_group.state = 'deleted'
+
+ package.package_group_all[:] = group_package_group.values()
+
-def relationship_list_save(relationship_dicts, context):
+def relationship_list_save(relationship_dicts, package, attr, context):
+ allow_partial_update = context.get("allow_partial_update", False)
+ if not relationship_dicts and allow_partial_update:
+ return
model = context["model"]
session = context["session"]
+ pending = context.get('pending')
- relationship_list = []
+ relationship_list = getattr(package, attr)
+ old_list = relationship_list[:]
+
+ relationships = []
for relationship_dict in relationship_dicts:
obj = table_dict_save(relationship_dict,
model.PackageRelationship, context)
- relationship_list.append(obj)
+ relationships.append(obj)
- return relationship_list
+ relationship_list[:] = relationships
+
+ for relationship in set(old_list) - set(relationship_list):
+ if pending and relationship.state <> 'deleted':
+ relationship.state = 'pending-deleted'
+ else:
+ relationship.state = 'deleted'
+ relationship_list.append(relationship)
def package_dict_save(pkg_dict, context):
@@ -117,33 +242,16 @@
pkg = table_dict_save(pkg_dict, Package, context)
- resources = resource_list_save(pkg_dict.get("resources", []), context)
- if resources:
- pkg.resources[:] = resources
+ package_resource_list_save(pkg_dict.get("resources", []), pkg, context)
+ package_tag_list_save(pkg_dict.get("tags", []), pkg, context)
+ package_group_list_save(pkg_dict.get("groups", []), pkg, context)
- tags = tag_list_save(pkg_dict.get("tags", []), context)
- if tags or not allow_partial_update:
- pkg.tags[:] = tags
+ subjects = pkg_dict.get('relationships_as_subject', [])
+ relationship_list_save(subjects, pkg, 'relationships_as_subject', context)
+ objects = pkg_dict.get('relationships_as_object', [])
+ relationship_list_save(subjects, pkg, 'relationships_as_object', context)
- groups = group_list_save(pkg_dict.get("groups", []), context)
- if groups or not allow_partial_update:
- pkg.groups[:] = groups
-
- subjects = pkg_dict.get("relationships_as_subject", [])
- if subjects or not allow_partial_update:
- pkg.relationships_as_subject[:] = relationship_list_save(subjects, context)
- objects = pkg_dict.get("relationships_as_object", [])
- if objects or not allow_partial_update:
- pkg.relationships_as_object[:] = relationship_list_save(objects, context)
-
- extras = extras_save(pkg_dict.get("extras", {}), context)
- if extras or not allow_partial_update:
- old_extras = set(pkg.extras.keys())
- new_extras = set(extras.keys())
- for key in old_extras - new_extras:
- del pkg.extras[key]
- for key in new_extras:
- pkg.extras[key] = extras[key]
+ extras = package_extras_save(pkg_dict.get("extras", []), pkg, context)
return pkg
@@ -161,7 +269,7 @@
group_dict["id"] = group.id
group = table_dict_save(group_dict, Group, context)
- extras = extras_save(group_dict.get("extras", {}), context)
+ extras = group_extras_save(group_dict.get("extras", {}), context)
if extras or not allow_partial_update:
old_extras = set(group.extras.keys())
new_extras = set(extras.keys())
--- a/ckan/lib/package_saver.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/lib/package_saver.py Thu Jun 23 12:30:51 2011 +0100
@@ -26,27 +26,32 @@
# TODO: rename to something more correct like prepare_for_render
@classmethod
- def render_package(cls, pkg):
+ def render_package(cls, pkg, context):
'''Prepares for rendering a package. Takes a Package object and
formats it for the various context variables required to call
render.
Note that the actual calling of render('package/read') is left
to the caller.'''
- c.pkg = pkg
try:
- notes_formatted = ckan.misc.MarkdownFormat().to_html(pkg.notes)
+ notes_formatted = ckan.misc.MarkdownFormat().to_html(pkg.get('notes',''))
c.pkg_notes_formatted = genshi.HTML(notes_formatted)
except Exception, e:
error_msg = "<span class='inline-warning'>%s</span>" % _("Cannot render package description")
c.pkg_notes_formatted = genshi.HTML(error_msg)
- c.current_rating, c.num_ratings = ckan.rating.get_rating(pkg)
- c.pkg_url_link = h.link_to(c.pkg.url, c.pkg.url, rel='foaf:homepage', target='_blank') \
- if c.pkg.url else _("No web page given")
- c.pkg_author_link = cls._person_email_link(c.pkg.author, c.pkg.author_email, "Author")
- c.pkg_maintainer_link = cls._person_email_link(c.pkg.maintainer, c.pkg.maintainer_email, "Maintainer")
- c.package_relationships = pkg.get_relationships_printable()
+ c.current_rating, c.num_ratings = ckan.rating.get_rating(context['package'])
+ url = pkg.get('url', '')
+ c.pkg_url_link = h.link_to(url, url, rel='foaf:homepage', target='_blank') \
+ if url else _("No web page given")
+ c.pkg_author_link = cls._person_email_link(pkg.get('author', ''), pkg.get('author_email', ''), "Author")
+ maintainer = pkg.get('maintainer', '')
+ maintainer_email = pkg.get('maintainer_email', '')
+ c.pkg_maintainer_link = cls._person_email_link(maintainer, maintainer_email, "Maintainer")
+ c.package_relationships = context['package'].get_relationships_printable()
c.pkg_extras = []
- for k, v in sorted(pkg.extras.items()):
+ for extra in sorted(pkg.get('extras',[]), key=lambda x:x['key']):
+ if extra.get('state') == 'deleted':
+ continue
+ k, v = extra['key'], extra['value']
if k in g.package_hide_extras:
continue
if isinstance(v, (list, tuple)):
--- a/ckan/logic/__init__.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/logic/__init__.py Thu Jun 23 12:30:51 2011 +0100
@@ -1,5 +1,6 @@
import logging
import ckan.authz
+from ckan.lib.navl.dictization_functions import flatten_dict
class ActionError(Exception):
def __init__(self, extra_msg=None):
@@ -65,6 +66,11 @@
data_dict[new_key] = value
return data_dict
+def flatten_to_string_key(dict):
+
+ flattented = flatten_dict(dict)
+ return untuplize_dict(flattented)
+
def check_access(entity, action, context):
model = context["model"]
user = context.get("user")
--- a/ckan/logic/action/create.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/logic/action/create.py Thu Jun 23 12:30:51 2011 +0100
@@ -71,7 +71,10 @@
## this is added so that the rest controller can make a new location
context["id"] = pkg.id
log.debug('Created object %s' % str(pkg.name))
- return package_dictize(pkg, context)
+ if not preview:
+ return package_dictize(pkg, context)
+ else:
+ return data
def resource_create(data_dict, context):
model = context['model']
--- a/ckan/logic/action/get.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/logic/action/get.py Thu Jun 23 12:30:51 2011 +0100
@@ -1,13 +1,16 @@
+from sqlalchemy.sql import select
from ckan.logic import NotFound, check_access
from ckan.plugins import (PluginImplementations,
IGroupController,
IPackageController)
import ckan.authz
+from ckan.lib.dictization import table_dictize
from ckan.lib.dictization.model_dictize import group_to_api1, group_to_api2
from ckan.lib.dictization.model_dictize import (package_to_api1,
package_to_api2,
package_dictize,
+ resource_list_dictize,
group_dictize)
@@ -21,6 +24,35 @@
packages = query.all()
return [getattr(p, ref_package_by) for p in packages]
+def current_package_list_with_resources(context):
+ model = context["model"]
+ user = context["user"]
+ limit = context.get("limit")
+
+ q = ckan.authz.Authorizer().authorized_query(user, model.PackageRevision)
+ q = q.order_by(model.package_revision_table.c.revision_timestamp.desc())
+ if limit:
+ q = q.limit(limit)
+ pack_rev = q.all()
+ package_list = []
+ for package in pack_rev:
+ result_dict = table_dictize(package, context)
+ res_rev = model.resource_revision_table
+ resource_group = model.resource_group_table
+ q = select([res_rev], from_obj = res_rev.join(resource_group,
+ resource_group.c.id == res_rev.c.resource_group_id))
+ q = q.where(resource_group.c.package_id == package.id)
+ result = q.where(res_rev.c.current == True).execute()
+ result_dict["resources"] = resource_list_dictize(result, context)
+ license_id = result_dict['license_id']
+ if license_id:
+ isopen = model.Package.get_license_register()[license_id].isopen()
+ result_dict['isopen'] = isopen
+ else:
+ result_dict['isopen'] = False
+ package_list.append(result_dict)
+ return package_list
+
def revision_list(context):
model = context["model"]
--- a/ckan/logic/action/update.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/logic/action/update.py Thu Jun 23 12:30:51 2011 +0100
@@ -1,5 +1,6 @@
import logging
import re
+import datetime
import ckan.authz
from ckan.plugins import PluginImplementations, IGroupController, IPackageController
@@ -70,8 +71,77 @@
for group in groups:
check_access(group, model.Action.EDIT, context)
+def _make_latest_rev_active(context, q):
+
+ session = context['model'].Session
+
+ old_current = q.filter_by(current=True).first()
+ if old_current:
+ old_current.current = False
+ session.add(old_current)
+
+ latest_rev = q.filter_by(expired_timestamp=datetime.datetime(9999, 12, 31)).one()
+ latest_rev.current = True
+ if latest_rev.state in ('pending-deleted', 'deleted'):
+ latest_rev.state = 'deleted'
+ else:
+ latest_rev.state = 'active'
+
+ session.add(latest_rev)
+
+ ##this is just a way to get the latest revision that changed
+ ##in order to timestamp
+ old_latest = context.get('latest_revision_date')
+ if old_latest:
+ if latest_rev.revision_timestamp > old_latest:
+ context['latest_revision_date'] = latest_rev.revision_timestamp
+ context['latest_revision'] = latest_rev.revision_id
+ else:
+ context['latest_revision_date'] = latest_rev.revision_timestamp
+ context['latest_revision'] = latest_rev.revision_id
+
+def make_latest_pending_package_active(context):
+
+ model = context['model']
+ session = model.Session
+ id = context["id"]
+ pkg = model.Package.get(id)
+
+ check_access(pkg, model.Action.EDIT, context)
+
+ #packages
+ q = session.query(model.PackageRevision).filter_by(id=pkg.id)
+ _make_latest_rev_active(context, q)
+
+ #resources
+ for resource in pkg.resource_groups_all[0].resources_all:
+ q = session.query(model.ResourceRevision).filter_by(id=resource.id)
+ _make_latest_rev_active(context, q)
+
+ #tags
+ for tag in pkg.package_tag_all:
+ q = session.query(model.PackageTagRevision).filter_by(id=tag.id)
+ _make_latest_rev_active(context, q)
+
+ #extras
+ for extra in pkg.extras_list:
+ q = session.query(model.PackageExtraRevision).filter_by(id=extra.id)
+ _make_latest_rev_active(context, q)
+
+ latest_revision = context.get('latest_revision')
+ if not latest_revision:
+ return
+
+ q = session.query(model.Revision).filter_by(id=latest_revision)
+ revision = q.first()
+ revision.approved_timestamp = datetime.datetime.now()
+ session.add(revision)
+
+ session.commit()
+ session.remove()
+
+
def package_update(data_dict, context):
-
model = context['model']
user = context['user']
id = context["id"]
@@ -86,10 +156,11 @@
raise NotFound(_('Package was not found.'))
check_access(pkg, model.Action.EDIT, context)
- check_group_auth(data_dict, context)
data, errors = validate(data_dict, schema, context)
+ check_group_auth(data, context)
+
if errors:
model.Session.rollback()
raise ValidationError(errors, package_error_summary(errors))
@@ -108,8 +179,8 @@
for item in PluginImplementations(IPackageController):
item.edit(pkg)
model.repo.commit()
-
- return package_dictize(pkg, context)
+ return package_dictize(pkg, context)
+ return data
def _update_package_relationship(relationship, comment, context):
@@ -159,8 +230,6 @@
comment = data_dict.get('comment', u'')
return _update_package_relationship(entity, comment, context)
-
-
def group_update(data_dict, context):
model = context['model']
--- a/ckan/logic/schema.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/logic/schema.py Thu Jun 23 12:30:51 2011 +0100
@@ -38,6 +38,7 @@
'hash': [ignore_missing, unicode],
'state': [ignore],
'position': [ignore],
+ 'revision_timestamp': [ignore],
'__extras': [ignore_missing, extras_unicode_convert, keep_extras],
}
@@ -51,6 +52,8 @@
tag_length_validator,
tag_name_validator,
tag_not_uppercase],
+ 'revision_timestamp': [ignore],
+ 'state': [ignore],
}
return schema
@@ -167,6 +170,7 @@
'value': [not_missing, unicode],
'state': [ignore],
'deleted': [ignore_missing],
+ 'revision_timestamp': [ignore],
}
return schema
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ckan/migration/versions/039_add_expired_id_and_dates.py Thu Jun 23 12:30:51 2011 +0100
@@ -0,0 +1,219 @@
+from migrate import *
+import uuid
+import datetime
+
+def upgrade(migrate_engine):
+
+ id = uuid.uuid4()
+
+ make_missing_revisions = '''
+
+-- make sure all tables have an entry in the revision_table
+
+insert into revision values ('%(id)s' , '%(timestamp)s', 'admin', 'Admin: make sure every object has a row in a revision table', 'active');
+
+insert into package_tag_revision (id,package_id,tag_id,revision_id,state,continuity_id) select id,package_id,tag_id, '%(id)s' ,state, id from package_tag where package_tag.id not in (select id from package_tag_revision);
+
+insert into resource_revision (id,resource_group_id,url,format,description,position,revision_id,hash,state,extras,continuity_id) select id,resource_group_id,url,format,description,position, '%(id)s' ,hash,state,extras, id from resource where resource.id not in (select id from resource_revision);
+
+insert into group_extra_revision (id,group_id,key,value,state,revision_id,continuity_id) select id,group_id,key,value,state, '%(id)s' , id from group_extra where group_extra.id not in (select id from group_extra_revision);
+
+insert into resource_group_revision (id,package_id,label,sort_order,extras,state,revision_id,continuity_id) select id,package_id,label,sort_order,extras,state, '%(id)s', id from resource_group where resource_group.id not in (select id from resource_group_revision);
+
+insert into package_extra_revision (id,package_id,key,value,revision_id,state,continuity_id) select id,package_id,key,value, '%(id)s',state, id from package_extra where package_extra.id not in (select id from package_extra_revision);
+
+insert into package_relationship_revision (id,subject_package_id,object_package_id,type,comment,revision_id,state,continuity_id) select id,subject_package_id,object_package_id,type,comment, '%(id)s',state, id from package_relationship where package_relationship.id not in (select id from package_relationship_revision);
+
+insert into group_revision (id,name,title,description,created,state,revision_id,continuity_id) select id,name,title,description,created,state, '%(id)s', id from "group" where "group".id not in (select id from group_revision);
+
+insert into package_revision (id,name,title,url,notes,license_id,revision_id,version,author,author_email,maintainer,maintainer_email,state,continuity_id) select id,name,title,url,notes,license_id, '%(id)s',version,author,author_email,maintainer,maintainer_email,state, id from package where package.id not in (select id from package_revision);
+
+''' % dict(id=id, timestamp=datetime.datetime.now().isoformat())
+
+
+ update_schema = '''
+ALTER TABLE package_revision
+ ADD COLUMN expired_id text,
+ ADD COLUMN revision_timestamp timestamp without time zone,
+ ADD COLUMN expired_timestamp timestamp without time zone,
+ ADD COLUMN current boolean;
+
+ALTER TABLE package_extra_revision
+ ADD COLUMN expired_id text,
+ ADD COLUMN revision_timestamp timestamp without time zone,
+ ADD COLUMN expired_timestamp timestamp without time zone,
+ ADD COLUMN current boolean;
+
+ALTER TABLE group_revision
+ ADD COLUMN expired_id text,
+ ADD COLUMN revision_timestamp timestamp without time zone,
+ ADD COLUMN expired_timestamp timestamp without time zone,
+ ADD COLUMN current boolean;
+
+ALTER TABLE group_extra_revision
+ ADD COLUMN expired_id text,
+ ADD COLUMN revision_timestamp timestamp without time zone,
+ ADD COLUMN expired_timestamp timestamp without time zone,
+ ADD COLUMN current boolean;
+
+
+ALTER TABLE package_group_revision
+ ADD COLUMN expired_id text,
+ ADD COLUMN revision_timestamp timestamp without time zone,
+ ADD COLUMN expired_timestamp timestamp without time zone,
+ ADD COLUMN current boolean;
+
+ALTER TABLE package_tag_revision
+ ADD COLUMN expired_id text,
+ ADD COLUMN revision_timestamp timestamp without time zone,
+ ADD COLUMN expired_timestamp timestamp without time zone,
+ ADD COLUMN current boolean;
+
+ALTER TABLE resource_group_revision
+ ADD COLUMN expired_id text,
+ ADD COLUMN revision_timestamp timestamp without time zone,
+ ADD COLUMN expired_timestamp timestamp without time zone,
+ ADD COLUMN current boolean;
+
+ALTER TABLE resource_revision
+ ADD COLUMN expired_id text,
+ ADD COLUMN revision_timestamp timestamp without time zone,
+ ADD COLUMN expired_timestamp timestamp without time zone,
+ ADD COLUMN current boolean;
+
+ALTER TABLE package_relationship_revision
+ ADD COLUMN expired_id text,
+ ADD COLUMN revision_timestamp timestamp without time zone,
+ ADD COLUMN expired_timestamp timestamp without time zone,
+ ADD COLUMN current boolean;
+
+ALTER TABLE revision
+ ADD COLUMN approved_timestamp timestamp without time zone;
+
+create table tmp_expired_id(id text, revision_id text, revision_timestamp timestamp, expired_timestamp timestamp, expired_id text);
+create index id_exp on tmp_expired_id(id, revision_id);
+
+--package revision
+truncate tmp_expired_id;
+insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_revision pr join revision r on pr.revision_id = r.id;
+update package_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id);
+update package_revision set current = '1' where expired_timestamp = '9999-12-31';
+
+create index idx_package_period on package_revision(revision_timestamp, expired_timestamp, id);
+create index idx_package_current on package_revision(current);
+
+--package extra revision
+truncate tmp_expired_id;
+insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_extra_revision pr join revision r on pr.revision_id = r.id;
+update package_extra_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id);
+update package_extra_revision set current = '1' where expired_timestamp = '9999-12-31';
+
+create index idx_package_extra_period on package_extra_revision(revision_timestamp, expired_timestamp, id);
+create index idx_package_extra_period_package on package_extra_revision(revision_timestamp, expired_timestamp, package_id);
+create index idx_package_extra_current on package_extra_revision(current);
+
+--package group revision
+truncate tmp_expired_id;
+insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_group_revision pr join revision r on pr.revision_id = r.id;
+update package_group_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id);
+update package_group_revision set current = '1' where expired_timestamp = '9999-12-31';
+
+create index idx_package_group_period_package_group on package_group_revision(revision_timestamp, expired_timestamp, package_id, group_id);
+create index idx_package_group_current on package_group_revision(current);
+
+
+-- package_tags
+truncate tmp_expired_id;
+insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_tag_revision pr join revision r on pr.revision_id = r.id;
+update package_tag_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id);
+update package_tag_revision set current = '1' where expired_timestamp = '9999-12-31';
+
+create index idx_period_package_tag on package_tag_revision(revision_timestamp, expired_timestamp, package_id, tag_id);
+create index idx_package_tag_current on package_tag_revision(current);
+
+-- package relationship
+truncate tmp_expired_id;
+insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_relationship_revision pr join revision r on pr.revision_id = r.id;
+update package_relationship_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id);
+update package_relationship_revision set current = '1' where expired_timestamp = '9999-12-31';
+
+create index idx_period_package_relationship on package_relationship_revision(revision_timestamp, expired_timestamp, object_package_id, subject_package_id);
+create index idx_package_relationship_current on package_relationship_revision(current);
+
+-- resource revision
+truncate tmp_expired_id;
+insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from resource_revision pr join revision r on pr.revision_id = r.id;
+update resource_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id);
+update resource_revision set current = '1' where expired_timestamp = '9999-12-31';
+
+create index idx_resource_period on resource_revision(revision_timestamp, expired_timestamp, id);
+create index idx_resource_period_resource_group on resource_revision(revision_timestamp, expired_timestamp, resource_group_id);
+create index idx_resource_current on resource_revision(current);
+
+-- resource group revision;
+truncate tmp_expired_id;
+insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from resource_group_revision pr join revision r on pr.revision_id = r.id;
+update resource_group_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id);
+update resource_group_revision set current = '1' where expired_timestamp = '9999-12-31';
+
+create index idx_resource_group_period on resource_group_revision(revision_timestamp, expired_timestamp, id);
+create index idx_resource_group_period_package on resource_group_revision(revision_timestamp, expired_timestamp, package_id);
+create index idx_resource_group_current on resource_group_revision(current);
+
+--group revision;
+truncate tmp_expired_id;
+insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from group_revision pr join revision r on pr.revision_id = r.id;
+update group_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id);
+update group_revision set current = '1' where expired_timestamp = '9999-12-31';
+
+create index idx_group_period on group_revision(revision_timestamp, expired_timestamp, id);
+create index idx_group_current on group_revision(current);
+
+--group extra revision
+truncate tmp_expired_id;
+insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from group_extra_revision pr join revision r on pr.revision_id = r.id;
+update group_extra_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id),
+ expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id);
+update group_extra_revision set current = '1' where expired_timestamp = '9999-12-31';
+
+create index idx_group_extra_period on group_extra_revision(revision_timestamp, expired_timestamp, id);
+create index idx_group_extra_period_group on group_extra_revision(revision_timestamp, expired_timestamp, group_id);
+create index idx_group_extra_current on group_extra_revision(current);
+
+drop table tmp_expired_id;
+
+-- change state of revision tables
+
+update revision set approved_timestamp = timestamp;
+'''
+
+ migrate_engine.execute('begin; ' + make_missing_revisions + update_schema + ' commit;')
+
+ for table in ['package', 'resource', 'resource_group', 'package_extra',
+ 'package_tag', 'package_relationship', 'group', 'group_extra']:
+ count = migrate_engine.execute('''select count(*) from "%s"''' % table).first()[0]
+ revision_expired_id_count = migrate_engine.execute('''select count(*) from %s_revision where %s_revision.expired_id is null''' % (table, table)).first()[0]
+ revision_expired_data_count = migrate_engine.execute('''select count(*) from %s_revision where %s_revision.expired_timestamp = '9999-12-31' ''' % (table, table)).first()[0]
+ revision_current = migrate_engine.execute('''select count(*) from %s_revision where %s_revision.current = '1' ''' % (table, table)).first()[0]
+ assert count == revision_expired_id_count
+ assert count == revision_expired_data_count
+ assert count == revision_current
+
+
--- a/ckan/model/changeset.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/changeset.py Thu Jun 23 12:30:51 2011 +0100
@@ -981,7 +981,7 @@
def get_columns(self):
"""Returns the model of the entity attributes."""
- from ckan.model.core import orm
+ from sqlalchemy import orm
table = orm.class_mapper(self.object_type).mapped_table
return table.c
--- a/ckan/model/core.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/core.py Thu Jun 23 12:30:51 2011 +0100
@@ -1,10 +1,12 @@
-from meta import *
+from meta import metadata, mapper
+from sqlalchemy import Column, DateTime, Text, Boolean
import vdm.sqlalchemy
from domain_object import DomainObject
## VDM-specific tables
revision_table = vdm.sqlalchemy.make_revision_table(metadata)
+revision_table.append_column(Column('approved_timestamp', DateTime))
class System(DomainObject):
@@ -26,5 +28,14 @@
Revision = vdm.sqlalchemy.make_Revision(mapper, revision_table)
+def make_revisioned_table(table):
+ import datetime
+ revision_table = vdm.sqlalchemy.make_revisioned_table(table)
+ revision_table.append_column(Column('expired_id',
+ Text))
+ revision_table.append_column(Column('revision_timestamp', DateTime))
+ revision_table.append_column(Column('expired_timestamp', DateTime,
+ default=datetime.datetime(9999, 12, 31)))
+ revision_table.append_column(Column('current', Boolean))
+ return revision_table
-
--- a/ckan/model/domain_object.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/domain_object.py Thu Jun 23 12:30:51 2011 +0100
@@ -1,8 +1,9 @@
import datetime
+from sqlalchemy import orm
from sqlalchemy.util import OrderedDict
-from meta import *
+from meta import Session
class Enum(set):
'''Simple enumeration
--- a/ckan/model/group.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/group.py Thu Jun 23 12:30:51 2011 +0100
@@ -11,7 +11,8 @@
from sqlalchemy.ext.associationproxy import association_proxy
__all__ = ['group_table', 'Group', 'package_revision_table',
- 'PackageGroup', 'GroupRevision', 'PackageGroupRevision']
+ 'PackageGroup', 'GroupRevision', 'PackageGroupRevision',
+ 'package_group_revision_table']
package_group_table = Table('package_group', metadata,
Column('id', UnicodeText, primary_key=True, default=make_uuid),
@@ -20,7 +21,7 @@
)
vdm.sqlalchemy.make_table_stateful(package_group_table)
-package_group_revision_table = vdm.sqlalchemy.make_revisioned_table(package_group_table)
+package_group_revision_table = make_revisioned_table(package_group_table)
group_table = Table('group', metadata,
Column('id', UnicodeText, primary_key=True, default=make_uuid),
@@ -31,7 +32,7 @@
)
vdm.sqlalchemy.make_table_stateful(group_table)
-group_revision_table = vdm.sqlalchemy.make_revisioned_table(group_table)
+group_revision_table = make_revisioned_table(group_table)
class PackageGroup(vdm.sqlalchemy.RevisionedObjectMixin,
--- a/ckan/model/group_extra.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/group_extra.py Thu Jun 23 12:30:51 2011 +0100
@@ -18,7 +18,7 @@
)
vdm.sqlalchemy.make_table_stateful(group_extra_table)
-group_extra_revision_table = vdm.sqlalchemy.make_revisioned_table(group_extra_table)
+group_extra_revision_table = make_revisioned_table(group_extra_table)
class GroupExtra(vdm.sqlalchemy.RevisionedObjectMixin,
--- a/ckan/model/meta.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/meta.py Thu Jun 23 12:30:51 2011 +0100
@@ -1,7 +1,9 @@
+import datetime
"""SQLAlchemy Metadata and Session object"""
from sqlalchemy import MetaData, __version__ as sqav
from sqlalchemy.orm import scoped_session, sessionmaker
import sqlalchemy.orm as orm
+from sqlalchemy.orm.session import SessionExtension
# TODO: remove these imports from here and put them in client model modules
from sqlalchemy import Column, MetaData, Table, types, ForeignKey
@@ -12,6 +14,74 @@
from ckan.model import extension
+class CkanSessionExtension(SessionExtension):
+
+ def before_flush(self, session, flush_context, instances):
+ if not hasattr(session, '_object_cache'):
+ session._object_cache= {'new': set(),
+ 'deleted': set(),
+ 'changed': set()}
+
+ changed = [obj for obj in session.dirty if
+ session.is_modified(obj, include_collections=False)]
+
+ session._object_cache['new'].update(session.new)
+ session._object_cache['deleted'].update(session.deleted)
+ session._object_cache['changed'].update(changed)
+
+
+ def before_commit(self, session):
+ session.flush()
+ try:
+ obj_cache = session._object_cache
+ revision = session.revision
+ except AttributeError:
+ return
+
+ new = obj_cache['new']
+ changed = obj_cache['changed']
+ deleted = obj_cache['deleted']
+
+ for obj in new | changed | deleted:
+
+ if not hasattr(obj, '__revision_class__'):
+ continue
+
+ revision_cls = obj.__revision_class__
+
+ ## when a normal active transaction happens
+ if 'pending' not in obj.state:
+ revision.approved_timestamp = datetime.datetime.now()
+ old = session.query(revision_cls).filter_by(
+ current='1',
+ id = obj.id
+ ).first()
+ if old:
+ old.current = '0'
+ session.add(old)
+
+ q = session.query(revision_cls)
+ q = q.filter_by(expired_timestamp=datetime.datetime(9999, 12, 31), id=obj.id)
+ results = q.all()
+
+ for rev_obj in results:
+ if rev_obj.revision_id == revision.id:
+ rev_obj.revision_timestamp = revision.timestamp
+ if 'pending' not in obj.state:
+ rev_obj.current = '1'
+ else:
+ rev_obj.expired_id = revision.id
+ rev_obj.expired_timestamp = revision.timestamp
+ session.add(rev_obj)
+
+ def after_commit(self, session):
+ if hasattr(session, '_object_cache'):
+ del session._object_cache
+
+ def after_rollback(self, session):
+ if hasattr(session, '_object_cache'):
+ del session._object_cache
+
# __all__ = ['Session', 'engine', 'metadata', 'mapper']
# SQLAlchemy database engine. Updated by model.init_model()
@@ -22,14 +92,16 @@
Session = scoped_session(sessionmaker(
autoflush=False,
transactional=True,
- extension=extension.PluginSessionExtension(),
+ extension=[CkanSessionExtension(),
+ extension.PluginSessionExtension()],
))
else:
Session = scoped_session(sessionmaker(
autoflush=False,
autocommit=False,
expire_on_commit=False,
- extension=extension.PluginSessionExtension(),
+ extension=[CkanSessionExtension(),
+ extension.PluginSessionExtension()],
))
#mapper = Session.mapper
--- a/ckan/model/modification.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/modification.py Thu Jun 23 12:30:51 2011 +0100
@@ -26,20 +26,6 @@
implements(ISession, inherit=True)
observers = PluginImplementations(IDomainObjectModification)
- def before_flush(self, session, flush_context, instances):
-
- if not hasattr(session, '_object_cache'):
- session._object_cache= {'new': set(),
- 'deleted': set(),
- 'changed': set()}
-
- changed = [obj for obj in session.dirty if
- session.is_modified(obj, include_collections=False)]
-
- session._object_cache['new'].update(session.new)
- session._object_cache['deleted'].update(session.deleted)
- session._object_cache['changed'].update(changed)
-
def before_commit(self, session):
session.flush()
@@ -66,6 +52,8 @@
related_packages = obj.related_packages()
except AttributeError:
continue
+ if 'pending' in obj.state:
+ continue
# this is needed to sort out vdm bug where pkg.as_dict does not
# work when the package is deleted.
for package in related_packages:
@@ -73,7 +61,6 @@
changed_pkgs.add(package)
for obj in changed_pkgs:
self.notify(obj, DomainObjectOperation.changed)
- del session._object_cache
def notify(self, entity, operation):
--- a/ckan/model/package.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/package.py Thu Jun 23 12:30:51 2011 +0100
@@ -2,14 +2,15 @@
from time import gmtime
from calendar import timegm
-from sqlalchemy.sql import select, and_, union, expression
+from sqlalchemy.sql import select, and_, union, expression, or_
from sqlalchemy.orm import eagerload_all
+from sqlalchemy import types, Column, Table
from pylons import config
-from meta import *
+from meta import metadata, Session
import vdm.sqlalchemy
from types import make_uuid
-from core import *
+from core import make_revisioned_table, Revision, State
from license import License, LicenseRegister
from domain_object import DomainObject
import ckan.misc
@@ -37,7 +38,7 @@
vdm.sqlalchemy.make_table_stateful(package_table)
-package_revision_table = vdm.sqlalchemy.make_revisioned_table(package_table)
+package_revision_table = make_revisioned_table(package_table)
## -------------------
## Mapped classes
--- a/ckan/model/package_extra.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/package_extra.py Thu Jun 23 12:30:51 2011 +0100
@@ -7,7 +7,8 @@
from types import JsonType
from ckan.model import extension
-__all__ = ['PackageExtra', 'package_extra_table', 'PackageExtraRevision']
+__all__ = ['PackageExtra', 'package_extra_table', 'PackageExtraRevision',
+ 'extra_revision_table']
package_extra_table = Table('package_extra', metadata,
Column('id', UnicodeText, primary_key=True, default=make_uuid),
@@ -18,7 +19,7 @@
)
vdm.sqlalchemy.make_table_stateful(package_extra_table)
-extra_revision_table= vdm.sqlalchemy.make_revisioned_table(package_extra_table)
+extra_revision_table= make_revisioned_table(package_extra_table)
class PackageExtra(vdm.sqlalchemy.RevisionedObjectMixin,
vdm.sqlalchemy.StatefulObjectMixin,
@@ -33,6 +34,11 @@
collection_class=orm.collections.attribute_mapped_collection(u'key'),
cascade='all, delete, delete-orphan',
),
+ ),
+ 'package_no_state': orm.relation(Package,
+ backref=orm.backref('extras_list',
+ cascade='all, delete, delete-orphan',
+ ),
)
},
order_by=[package_extra_table.c.package_id, package_extra_table.c.key],
--- a/ckan/model/package_relationship.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/package_relationship.py Thu Jun 23 12:30:51 2011 +0100
@@ -24,7 +24,7 @@
)
vdm.sqlalchemy.make_table_stateful(package_relationship_table)
-package_relationship_revision_table = vdm.sqlalchemy.make_revisioned_table(package_relationship_table)
+package_relationship_revision_table = make_revisioned_table(package_relationship_table)
class PackageRelationship(vdm.sqlalchemy.RevisionedObjectMixin,
vdm.sqlalchemy.StatefulObjectMixin,
--- a/ckan/model/resource.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/resource.py Thu Jun 23 12:30:51 2011 +0100
@@ -39,10 +39,10 @@
)
vdm.sqlalchemy.make_table_stateful(resource_table)
-resource_revision_table = vdm.sqlalchemy.make_revisioned_table(resource_table)
+resource_revision_table = make_revisioned_table(resource_table)
vdm.sqlalchemy.make_table_stateful(resource_group_table)
-resource_group_revision_table = vdm.sqlalchemy.make_revisioned_table(resource_group_table)
+resource_group_revision_table = make_revisioned_table(resource_group_table)
class Resource(vdm.sqlalchemy.RevisionedObjectMixin,
vdm.sqlalchemy.StatefulObjectMixin,
--- a/ckan/model/tag.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/model/tag.py Thu Jun 23 12:30:51 2011 +0100
@@ -8,7 +8,7 @@
from core import *
__all__ = ['tag_table', 'package_tag_table', 'Tag', 'PackageTag',
- 'PackageTagRevision']
+ 'PackageTagRevision', 'package_tag_revision_table']
tag_table = Table('tag', metadata,
Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
@@ -23,7 +23,7 @@
vdm.sqlalchemy.make_table_stateful(package_tag_table)
# TODO: this has a composite primary key ...
-package_tag_revision_table = vdm.sqlalchemy.make_revisioned_table(package_tag_table)
+package_tag_revision_table = make_revisioned_table(package_tag_table)
class Tag(DomainObject):
def __init__(self, name=''):
@@ -103,12 +103,15 @@
mapper(Tag, tag_table, properties={
'package_tags':relation(PackageTag, backref='tag',
cascade='all, delete, delete-orphan',
- )
+ ),
},
order_by=tag_table.c.name,
)
mapper(PackageTag, package_tag_table, properties={
+ 'pkg':relation(Package, backref='package_tag_all',
+ cascade='none',
+ )
},
order_by=package_tag_table.c.id,
extension=[vdm.sqlalchemy.Revisioner(package_tag_revision_table),
--- a/ckan/templates/_util.html Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/templates/_util.html Thu Jun 23 12:30:51 2011 +0100
@@ -27,7 +27,7 @@
tag listing --><ul py:def="tag_list(tags)" class="tags clearfix"><li py:for="tag in tags">
- ${h.link_to(tag.name, h.url_for(controller='tag', action='read', id=tag.name))}
+ ${h.link_to(tag['name'], h.url_for(controller='tag', action='read', id=tag['name']))}
</li></ul>
@@ -91,6 +91,56 @@
</li></ul>
+ <ul py:def="package_list_from_dict(packages)" class="packages">
+ <li py:for="package in packages"
+ class="${'fullyopen' if (package.isopen and package.get('resources')) else None}">
+ <div class="header">
+ <span class="title">
+ ${h.link_to(package.get('title') or package.get('name'), h.url_for(controller='package', action='read', id=package.get('name')))}
+ </span>
+
+ <div class="search_meta">
+ <py:if test="package.resources">
+ <ul class="package_formats">
+ <py:for each="resource in package.resources">
+ <py:if test="resource.get('format')">
+ <li>${resource.get('format')}</li>
+ </py:if>
+ </py:for>
+ </ul>
+ </py:if>
+ <ul class="openness">
+ <py:if test="package.isopen">
+ <li>
+ <a href="http://opendefinition.org/okd/" title="This package satisfies the Open Definition.">
+ <img src="http://assets.okfn.org/images/ok_buttons/od_80x15_blue.png" alt="[Open Data]" />
+ </a>
+ </li>
+ <li>
+ <a href="http://opendefinition.org/okd/" title="This package satisfies the Open Definition.">
+ <img src="http://assets.okfn.org/images/ok_buttons/oc_80x15_blue.png" alt="[Open Content]" />
+ </a>
+ </li>
+ </py:if>
+ <py:if test="not package.isopen">
+ <li>
+ <span class="closed">
+ ${h.icon('lock')} Not Openly Licensed
+ </span>
+ </li>
+ </py:if>
+ </ul>
+ </div>
+ </div>
+ <div class="extract">
+ ${h.markdown_extract(package.notes)}
+ </div>
+ <!--ul py:if="package.tags" class="tags">
+ <li py:for="tag in package.tags">${tag.name}</li>
+ </ul-->
+ </li>
+ </ul>
+
<!--! List of data package groups: pass in a collection of data package groups
and this renders the standard group listing --><table py:def="group_list(groups)" class="groups">
--- a/ckan/templates/home/index.html Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/templates/home/index.html Thu Jun 23 12:30:51 2011 +0100
@@ -64,7 +64,7 @@
</py:if><h4>Recently changed packages</h4>
- ${package_list(c.latest_packages)}
+ ${package_list_from_dict(c.latest_packages)}
<p><a href="${h.url_for(controller='revision', action='index',
id=None)}">View revision log »</a></p>
--- a/ckan/templates/package/comments.html Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/templates/package/comments.html Thu Jun 23 12:30:51 2011 +0100
@@ -2,12 +2,12 @@
xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">
- <py:def function="page_title">${c.pkg.title or c.pkg.name} - Data Packages - History</py:def>
+ <py:def function="page_title">${c.pkg_dict.get('title', c.pkg_dict['name'])} - Data Packages - History</py:def><div py:match="content" class="package"><h2 class="head">
- ${c.pkg.title}
- <span class="name">(${c.pkg.name})</span>
+ ${c.pkg_dict.get('title', '')}
+ <span class="name">(${c.pkg_dict['name']})</span></h2></div><!-- content -->
--- a/ckan/templates/package/new_package_form.html Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/templates/package/new_package_form.html Thu Jun 23 12:30:51 2011 +0100
@@ -13,7 +13,7 @@
</ul></div>
-<fieldset>
+<fieldset id="basic-information"><legend> Basic information</legend><dl><dt><label class="field_opt" for="title">Title</label></dt>
@@ -63,7 +63,7 @@
</dl></fieldset>
-<fieldset>
+<fieldset id="resources"><legend>Resources</legend><table class="flexitable"><thead>
@@ -94,7 +94,7 @@
<div class="field_error" py:if="errors.get('resources', '')">Package resource(s) incomplete.</div></fieldset>
-<fieldset>
+<fieldset id="groups"><legend>Groups</legend><dl><py:for each="num, group in enumerate(data.get('groups', []))">
@@ -117,7 +117,7 @@
<dd py:if="not c.groups">Cannot add any groups.</dd></dl></fieldset>
-<fieldset>
+<fieldset id='detail'><legend>Detail</legend><dl><dt><label class="field_opt" for="author">Author</label></dt>
@@ -150,7 +150,7 @@
</dl></fieldset>
-<fieldset>
+<fieldset id='extras'><legend>Extras</legend><dl><py:with vars="extras = data.get('extras', [])">
--- a/ckan/templates/package/read.html Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/templates/package/read.html Thu Jun 23 12:30:51 2011 +0100
@@ -3,7 +3,7 @@
xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">
- <py:def function="page_title">${c.pkg.title or c.pkg.name} - Data Packages</py:def>
+ <py:def function="page_title">${c.pkg_dict.get('title', c.pkg_dict['name'])} - Data Packages</py:def><py:match path="primarysidebar">
@@ -16,7 +16,7 @@
<li class="widget-container widget_text"><h3>Tags</h3>
- ${tag_list(c.pkg.tags)}
+ ${tag_list(c.pkg_dict.get('tags', ''))}
<p class="widget_action" py:if="h.am_authorized(c, actions.EDIT, c.pkg)">
${h.subnav_link(c, 'Add a new Tag', controller='package', action='edit', id=c.pkg.name)}
</p>
--- a/ckan/templates/package/read_core.html Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/templates/package/read_core.html Thu Jun 23 12:30:51 2011 +0100
@@ -6,7 +6,7 @@
<div id="package" class="package"><!-- Title --><h2 class="head">
- ${c.pkg.title}
+ ${c.pkg_dict.get('title','')}
<p class="atom-feed-link package-history-link"><a
href="${url(controller='package', action='history', id=c.pkg.name, format='atom', days=7)}"
@@ -16,7 +16,7 @@
</h2><!-- Source URL -->
- <div class="url" py:if="c.pkg.url">
+ <div class="url" py:if="c.pkg_dict.get('url')"><p>
Source: ${c.pkg_url_link}
</p>
@@ -31,30 +31,30 @@
<div class="resources subsection"><h3>Downloads & Resources</h3><py:choose test="">
- <table py:when="c.pkg.resources">
+ <table py:when="c.pkg_dict.get('resources', [])"><tr><th>Description</th><th>Format</th><th>Hash</th></tr>
- <py:for each="res in c.pkg.resources">
+ <py:for each="res in c.pkg_dict.get('resources', [])"><tr><td><py:choose test="">
- <py:when test="res.description">
- <a href="${res.url}" target="_blank">${res.description}</a>
+ <py:when test="res.get('description')">
+ <a href="${res.get('url', '')}" target="_blank">${res.description}</a></py:when><py:otherwise test="">
- <a href="${res.url}" target="_blank">Download <em>(no description)</em></a>
+ <a href="${res.get('url', '')}" target="_blank">Download <em>(no description)</em></a></py:otherwise></py:choose></td>
- <td>${res.format}</td>
- <td>${res.hash}</td>
+ <td>${res.get('format', '')}</td>
+ <td>${res.get('hash', '')}</td></tr></py:for><caption>
- This is a list of all known formats and datasets for <em>${c.pkg.title}</em>. If you know of another (CSV, SPARQL end-point etc.) ${h.subnav_link(c, 'please edit this page and add it to the list', controller='package', action='edit', id=c.pkg.name)}.
+ This is a list of all known formats and datasets for <em>${c.pkg_dict.get('title', '')}</em>. If you know of another (CSV, SPARQL end-point etc.) ${h.subnav_link(c, 'please edit this page and add it to the list', controller='package', action='edit', id=c.pkg.name)}.
</caption></table><table py:otherwise=""><tr><th>Resources</th><td>None given for this package.</td></tr></table>
@@ -80,7 +80,7 @@
<tbody>
${details_item('Author', c.pkg_author_link)}
${details_item('Maintainer', c.pkg_maintainer_link)}
- ${details_item('Version', c.pkg.version)}
+ ${details_item('Version', c.pkg_dict.get('version', ''))}
<tr><td class="package-label">
License
@@ -147,7 +147,7 @@
<ul><py:if test="h.am_authorized(c, actions.CHANGE_STATE, c.pkg)">
- <li><strong>State:</strong> ${c.pkg.state}</li>
+ <li><strong>State:</strong> ${c.pkg_dict.get('state', '')}</li></py:if></ul>
--- a/ckan/tests/functional/api/model/test_package.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/tests/functional/api/model/test_package.py Thu Jun 23 12:30:51 2011 +0100
@@ -260,6 +260,7 @@
# - url
self.assert_equal(package.url, self.package_fixture_data['url'])
# - extras
+
self.assert_equal(len(package.extras), 4)
for key, value in {u'key1':u'val1',
u'key3':u'val3',
--- a/ckan/tests/functional/test_authz.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/tests/functional/test_authz.py Thu Jun 23 12:30:51 2011 +0100
@@ -188,6 +188,7 @@
res = func(offset, params=postparams,
extra_environ=environ,
expect_errors=True)
+
tests = {}
tests['str_required (%s)' % str_required_in_response] = bool(str_required_in_response in res)
tests['error string'] = bool('error' not in res)
@@ -313,7 +314,7 @@
cls.groupreader = model.User.by_name(u'groupreader')
cls.mrloggedin = model.User.by_name(name=u'mrloggedin')
cls.visitor = model.User.by_name(name=model.PSEUDO_USER__VISITOR)
-
+
# Tests numbered by the use case
def test_14_visitor_reads_stopped(self):
--- a/ckan/tests/functional/test_package.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/tests/functional/test_package.py Thu Jun 23 12:30:51 2011 +0100
@@ -294,6 +294,7 @@
for txt in txt_order_non_deterministic:
for pkg_ in (pkg_by_name_main, pkg_by_id_main):
pkg_ = pkg_.replace(txt, 'placeholder')
+ print pkg_by_name_main
res_diff = self.diff_html(pkg_by_name_main, pkg_by_id_main)
assert not res_diff, res_diff.encode('utf8')
# not true as language selection link return url differs:
@@ -886,7 +887,7 @@
assert len(pkg.groups) == 0
grp = model.Group.by_name(u'david')
model.repo.new_revision()
- pkg.groups.append(grp)
+ model.Session.add(model.PackageGroup(package=pkg, group=grp))
model.repo.commit_and_remove()
pkg = model.Package.by_name(u'editpkgtest')
assert len(pkg.groups) == 1
@@ -895,8 +896,10 @@
prefix = ''
field_name = prefix + "groups__0__id"
fv = res.forms['package-edit']
+ print field_name
fv[field_name] = False
res = fv.submit('save', extra_environ={'REMOTE_USER':'russianfan'})
+ model.repo.commit_and_remove()
pkg = model.Package.by_name(u'editpkgtest')
assert len(pkg.groups) == 0
finally:
--- a/ckan/tests/lib/test_dictization.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/tests/lib/test_dictization.py Thu Jun 23 12:30:51 2011 +0100
@@ -13,30 +13,84 @@
package_to_api1,
package_to_api2,
)
-
from ckan.lib.dictization.model_save import (package_dict_save,
resource_dict_save,
group_dict_save,
package_api_to_dict,
group_api_to_dict,
)
+from ckan.logic.action.update import make_latest_pending_package_active
class TestBasicDictize:
@classmethod
def setup_class(cls):
CreateTestData.create()
+ cls.package_expected = {
+ 'author': None,
+ 'author_email': None,
+ 'extras': [
+ {'key': u'genre',
+ 'state': u'active',
+ 'value': '"romantic novel"'},
+ {'key': u'original media', 'state': u'active', 'value': u'"book"'}],
+ 'groups': [{'description': u'These are books that David likes.',
+ 'name': u'david',
+ 'state': u'active',
+ 'title': u"Dave's books"},
+ {'description': u'Roger likes these books.',
+ 'name': u'roger',
+ 'state': u'active',
+ 'title': u"Roger's books"}],
+ 'license_id': u'other-open',
+ 'maintainer': None,
+ 'maintainer_email': None,
+ 'name': u'annakarenina',
+ 'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
+ 'relationships_as_object': [],
+ 'relationships_as_subject': [],
+ 'resources': [{'alt_url': u'alt123',
+ 'description': u'Full text. Needs escaping: " Umlaut: \xfc',
+ 'size': u'123',
+ 'format': u'plain text',
+ 'hash': u'abc123',
+ 'position': 0,
+ 'state': u'active',
+ 'url': u'http://www.annakarenina.com/download/x=1&y=2'},
+ {'alt_url': u'alt345',
+ 'description': u'Index of the novel',
+ 'size': u'345',
+ 'format': u'json',
+ 'hash': u'def456',
+ 'position': 1,
+ 'state': u'active',
+ 'url': u'http://www.annakarenina.com/index.json'}],
+ 'state': u'active',
+ 'tags': [{'name': u'russian', 'state': u'active'},
+ {'name': u'tolstoy', 'state': u'active'}],
+ 'title': u'A Novel By Tolstoy',
+ 'url': u'http://www.annakarenina.com',
+ 'version': u'0.7a'}
+
+
@classmethod
def teardown_class(cls):
model.repo.rebuild_db()
model.Session.remove()
+ def teardonwn(self):
+ model.Session.remove()
+
+
+
def remove_changable_columns(self, dict):
for key, value in dict.items():
if key.endswith('id') and key <> 'license_id':
dict.pop(key)
if key == 'created':
dict.pop(key)
+ if 'timestamp' in key:
+ dict.pop(key)
if isinstance(value, list):
for new_dict in value:
self.remove_changable_columns(new_dict)
@@ -44,7 +98,8 @@
def remove_revision_id(self, dict):
for key, value in dict.items():
- if key == 'revision_id':
+ if key in ('revision_id', 'revision_timestamp',
+ 'expired_timestamp', 'expired_id'):
dict.pop(key)
if isinstance(value, list):
for new_dict in value:
@@ -54,10 +109,9 @@
def test_01_dictize_main_objects_simple(self):
context = {"model": model,
- "session": model.Session}
+ "session": model.Session}
## package
-
pkg = model.Session.query(model.Package).filter_by(name='annakarenina').first()
result = table_dictize(pkg, context)
self.remove_changable_columns(result)
@@ -83,6 +137,7 @@
result = resource_dictize(resource, context)
self.remove_changable_columns(result)
+
assert result == {
'alt_url': u'alt123',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
@@ -112,52 +167,17 @@
context = {"model": model,
"session": model.Session}
+ model.Session.remove()
pkg = model.Session.query(model.Package).filter_by(name='annakarenina').first()
result = package_dictize(pkg, context)
self.remove_changable_columns(result)
-
- assert result ==\
- {'author': None,
- 'author_email': None,
- 'extras': [{'key': u'original media', 'state': u'active', 'value': u'"book"'}],
- 'groups': [{'description': u'These are books that David likes.',
- 'name': u'david',
- 'state': u'active',
- 'title': u"Dave's books"},
- {'description': u'Roger likes these books.',
- 'name': u'roger',
- 'state': u'active',
- 'title': u"Roger's books"}],
- 'license_id': u'other-open',
- 'maintainer': None,
- 'maintainer_email': None,
- 'name': u'annakarenina',
- 'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n \nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
- 'relationships_as_object': [],
- 'relationships_as_subject': [],
- 'resources': [{'alt_url': u'alt123',
- 'description': u'Full text. Needs escaping: " Umlaut: \xfc',
- 'size': u'123',
- 'format': u'plain text',
- 'hash': u'abc123',
- 'position': 0,
- 'state': u'active',
- 'url': u'http://www.annakarenina.com/download/x=1&y=2'},
- {'alt_url': u'alt345',
- 'description': u'Index of the novel',
- 'size': u'345',
- 'format': u'json',
- 'hash': u'def456',
- 'position': 1,
- 'state': u'active',
- 'url': u'http://www.annakarenina.com/index.json'}],
- 'state': u'active',
- 'tags': [{'name': u'russian'}, {'name': u'tolstoy'}],
- 'title': u'A Novel By Tolstoy',
- 'url': u'http://www.annakarenina.com',
- 'version': u'0.7a'}, pprint(result)
+ pprint(result)
+ pprint(self.package_expected)
+
+ assert sorted(result.values()) == sorted(self.package_expected.values())
+ assert result == self.package_expected
@@ -257,10 +277,12 @@
def test_08_package_save(self):
context = {"model": model,
- "session": model.Session}
+ "session": model.Session}
anna1 = model.Session.query(model.Package).filter_by(name='annakarenina').one()
+
+
anna_dictized = self.remove_changable_columns(package_dictize(anna1, context))
anna_dictized["name"] = u'annakarenina3'
@@ -275,12 +297,10 @@
anna_original = pformat(anna_dictized)
anna_after_save = pformat(package_dictized)
- print anna_original
- print anna_after_save
assert self.remove_changable_columns(package_dictize(pkg, context)) == anna_dictized, "\n".join(unified_diff(anna_original.split("\n"), anna_after_save.split("\n")))
- def test_10_package_alter(self):
+ def test_09_package_alter(self):
context = {"model": model,
"session": model.Session}
@@ -301,6 +321,13 @@
package_dictized = package_dictize(pkg, context)
+ resources_revisions = model.Session.query(model.ResourceRevision).filter_by(resource_group_id=anna1.resource_groups[0].id).all()
+
+ sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1]
+ for res in sorted_resources:
+ print res.id, res.revision_timestamp, res.expired_timestamp, res.state, res.current
+ assert len(sorted_resources) == 3
+
anna_original = pformat(anna_dictized)
anna_after_save = pformat(package_dictized)
@@ -310,8 +337,318 @@
assert self.remove_revision_id(anna_dictized) == self.remove_revision_id(package_dictized),\
"\n".join(unified_diff(anna_original.split("\n"), anna_after_save.split("\n")))
+ def test_10_package_alter_pending(self):
- def test_11_resource_no_id(self):
+ context = {'model': model,
+ 'session': model.Session,
+ 'pending': True}
+
+ anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed').one()
+
+ anna_dictized = package_dictize(anna1, context)
+
+ anna_dictized['name'] = u'annakarenina_changed2'
+ anna_dictized['resources'][0]['url'] = u'new_url2'
+ anna_dictized['tags'][0]['name'] = u'new_tag'
+ anna_dictized['tags'][0].pop('id') #test if
+ anna_dictized['extras'][0]['value'] = u'"new_value"'
+
+ model.repo.new_revision()
+ package_dict_save(anna_dictized, context)
+ model.Session.commit()
+ model.Session.remove()
+
+ pkgrevisions = model.Session.query(model.PackageRevision).filter_by(id=anna1.id).all()
+
+ sorted_packages = sorted(pkgrevisions, key=lambda x:x.revision_timestamp)[::-1]
+
+ assert len(sorted_packages) == 3
+ assert sorted_packages[0].state == 'pending'
+ assert sorted_packages[1].state == 'active'
+ assert sorted_packages[1].current
+ assert sorted_packages[2].state == 'active'
+
+ assert str(sorted_packages[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_packages[1].expired_timestamp) != '9999-12-31 00:00:00'
+ assert str(sorted_packages[2].expired_timestamp) != '9999-12-31 00:00:00'
+
+ resources_revisions = model.Session.query(model.ResourceRevision).filter_by(resource_group_id=anna1.resource_groups[0].id).all()
+ sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1]
+
+ for pkg in sorted_resources:
+ print pkg.url, pkg.id, pkg.revision_timestamp, pkg.expired_timestamp, pkg.state, pkg.current
+
+ assert len(sorted_resources) == 4
+ assert sorted_resources[0].state == 'pending'
+ assert sorted_resources[1].state == 'active'
+ assert sorted_resources[1].current
+ assert sorted_resources[2].state == 'active'
+ assert sorted_resources[2].current
+ assert sorted_resources[3].state == 'active'
+
+ assert str(sorted_resources[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_resources[1].expired_timestamp) != '9999-12-31 00:00:00'
+ assert str(sorted_resources[2].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_resources[3].expired_timestamp) != '9999-12-31 00:00:00'
+
+ tag_revisions = model.Session.query(model.PackageTagRevision).filter_by(package_id=anna1.id).all()
+
+ sorted_tags = sorted(tag_revisions, key=lambda x: (x.revision_timestamp, x.tag.name))[::-1]
+
+ print [(tag.state, tag.tag.name) for tag in sorted_tags]
+
+ assert len(sorted_tags) == 4, len(sorted_tags)
+ assert sorted_tags[0].state == 'pending-deleted'
+ assert sorted_tags[1].state == 'pending'
+ assert sorted_tags[2].state == 'active'
+ assert sorted_resources[2].current
+ assert sorted_tags[3].state == 'active'
+
+ assert str(sorted_tags[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[1].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[2].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[3].expired_timestamp) != '9999-12-31 00:00:00'
+
+ extras_revisions = model.Session.query(model.PackageExtraRevision).filter_by(package_id=anna1.id).all()
+
+ sorted_extras = sorted(extras_revisions,
+ key=lambda x: (x.revision_timestamp, x.key))[::-1]
+
+ assert sorted_extras[0].state == 'pending'
+ assert sorted_resources[1].current
+ assert sorted_extras[1].state == 'active'
+ assert sorted_resources[1].current
+ assert sorted_extras[2].state == 'active'
+
+ assert str(sorted_extras[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_extras[1].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_extras[2].expired_timestamp) != '9999-12-31 00:00:00'
+
+
+ def test_11_add_pending(self):
+
+ context = {'model': model,
+ 'session': model.Session,
+ 'pending': True}
+
+ anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one()
+ anna_dictized = package_dictize(anna1, context)
+
+
+ anna_dictized['notes'] = 'wee'
+ anna_dictized['resources'].append({
+ 'format': u'plain text',
+ 'url': u'newurl'}
+ )
+ anna_dictized['tags'].append({'name': u'newnew_tag'})
+ anna_dictized['extras'].append({'key': 'david',
+ 'value': u'"new_value"'})
+
+ model.repo.new_revision()
+ package_dict_save(anna_dictized, context)
+ model.Session.commit()
+ model.Session.remove()
+
+ resources_revisions = model.Session.query(model.ResourceRevision).filter_by(resource_group_id=anna1.resource_groups[0].id).all()
+
+ sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1]
+ pprint(anna_dictized['resources'])
+
+ for pkg in sorted_resources:
+ print pkg.url, pkg.id, pkg.revision_timestamp, pkg.expired_timestamp, pkg.state, pkg.current
+
+
+ assert len(sorted_resources) == 5, len(sorted_resources)
+ assert sorted_resources[0].state == 'pending'
+ assert sorted_resources[1].state == 'pending'
+ assert sorted_resources[2].current
+ assert sorted_resources[2].state == 'active'
+ assert sorted_resources[3].current
+ assert sorted_resources[3].state == 'active'
+ assert sorted_resources[4].state == 'active'
+
+ assert str(sorted_resources[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_resources[1].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_resources[2].expired_timestamp) != '9999-12-31 00:00:00'
+ assert str(sorted_resources[3].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_resources[4].expired_timestamp) != '9999-12-31 00:00:00'
+
+ tag_revisions = model.Session.query(model.PackageTagRevision).filter_by(package_id=anna1.id).all()
+
+ sorted_tags = sorted(tag_revisions, key=lambda x: (x.revision_timestamp, x.tag.name))[::-1]
+
+ print [(tag.state, tag.tag.name) for tag in sorted_tags]
+
+ assert len(sorted_tags) == 5, len(sorted_tags)
+ assert sorted_tags[0].state == 'pending'
+ assert sorted_tags[1].state == 'pending-deleted'
+ assert sorted_tags[2].state == 'pending'
+ assert sorted_tags[3].state == 'active'
+ assert sorted_tags[3].current
+ assert sorted_tags[4].state == 'active'
+ assert sorted_tags[4].current
+
+ assert str(sorted_tags[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[1].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[2].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[3].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[4].expired_timestamp) != '9999-12-31 00:00:00'
+
+ extras_revisions = model.Session.query(model.PackageExtraRevision).filter_by(package_id=anna1.id).all()
+
+ sorted_extras = sorted(extras_revisions,
+ key=lambda x: (x.revision_timestamp, x.key))[::-1]
+
+ print [(extra.state, extra.key, extra.value) for extra in sorted_extras]
+
+ assert sorted_extras[0].state == 'pending'
+ assert sorted_extras[1].state == 'pending'
+ assert sorted_extras[2].state == 'active'
+ assert sorted_extras[3].state == 'active'
+
+ assert str(sorted_extras[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_extras[1].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_extras[2].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_extras[3].expired_timestamp) != '9999-12-31 00:00:00'
+
+ def test_12_make_active(self):
+
+ anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one()
+ context = {"model": model,
+ "session": model.Session,
+ 'user': 'testsysadmin',
+ "id": anna1.id}
+ make_latest_pending_package_active(context)
+
+ pkgrevisions = model.Session.query(model.PackageRevision).filter_by(id=anna1.id).all()
+ sorted_packages = sorted(pkgrevisions, key=lambda x:x.revision_timestamp)[::-1]
+
+ assert len(sorted_packages) == 4
+ assert sorted_packages[0].state == 'active', sorted_packages[0].state #was pending
+ assert sorted_packages[0].current == True
+
+ assert sorted_packages[1].state == 'pending'
+ assert sorted_packages[2].state == 'active'
+ assert sorted_packages[3].state == 'active'
+
+ resources_revisions = model.Session.query(model.ResourceRevision).filter_by(resource_group_id=anna1.resource_groups[0].id).all()
+ sorted_resources = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1]
+
+ assert len(sorted_resources) == 5
+ for res in sorted_resources:
+ print res.id, res.revision_timestamp, res.expired_timestamp, res.state
+ assert sorted_resources[0].state == 'active'
+ assert sorted_resources[0].current == True
+ assert sorted_resources[1].state == 'active'
+ assert sorted_resources[1].current == True
+ assert sorted_resources[2].state == 'active'
+ assert sorted_resources[3].state == 'active'
+ assert sorted_resources[3].current == True
+ assert sorted_resources[4].state == 'active'
+
+ assert str(sorted_resources[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_resources[1].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_resources[2].expired_timestamp) != '9999-12-31 00:00:00'
+ assert str(sorted_resources[3].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_resources[4].expired_timestamp) != '9999-12-31 00:00:00'
+
+ tag_revisions = model.Session.query(model.PackageTagRevision).filter_by(package_id=anna1.id).all()
+
+ sorted_tags = sorted(tag_revisions, key=lambda x: (x.revision_timestamp, x.tag.name))[::-1]
+
+ print [(tag.state, tag.tag.name) for tag in sorted_tags]
+
+ assert len(sorted_tags) == 5, len(sorted_tags)
+ assert sorted_tags[0].state == 'active'
+ assert sorted_tags[0].current
+ assert sorted_tags[1].state == 'deleted'
+ assert sorted_tags[1].current
+ assert sorted_tags[2].state == 'active'
+ assert sorted_tags[2].current
+ assert sorted_tags[3].state == 'active'
+ assert sorted_tags[4].state == 'active'
+
+ assert str(sorted_tags[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[1].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[2].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[3].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_tags[4].expired_timestamp) != '9999-12-31 00:00:00'
+
+ extras_revisions = model.Session.query(model.PackageExtraRevision).filter_by(package_id=anna1.id).all()
+
+ sorted_extras = sorted(extras_revisions,
+ key=lambda x: (x.revision_timestamp, x.key))[::-1]
+
+ print [(extra.state, extra.key, extra.value) for extra in sorted_extras]
+
+ assert sorted_extras[0].state == 'active'
+ assert sorted_extras[1].state == 'active'
+ assert sorted_extras[2].state == 'active'
+ assert sorted_extras[3].state == 'active'
+
+ assert str(sorted_extras[0].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_extras[1].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_extras[2].expired_timestamp) == '9999-12-31 00:00:00'
+ assert str(sorted_extras[3].expired_timestamp) != '9999-12-31 00:00:00'
+
+ def test_13_get_package_in_past(self):
+
+ context = {'model': model,
+ 'session': model.Session}
+
+ anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one()
+
+ pkgrevisions = model.Session.query(model.PackageRevision).filter_by(id=anna1.id).all()
+ sorted_packages = sorted(pkgrevisions, key=lambda x:x.revision_timestamp)
+
+ context['revision_id'] = sorted_packages[0].revision_id #original state
+
+ first_dictized = self.remove_changable_columns(package_dictize(anna1, context))
+ assert self.package_expected == first_dictized
+
+ context['revision_id'] = sorted_packages[1].revision_id #original state
+
+ second_dictized = self.remove_changable_columns(package_dictize(anna1, context))
+
+ first_dictized["name"] = u'annakarenina_changed'
+ first_dictized["resources"][0]["url"] = u'new_url'
+
+ assert second_dictized == first_dictized
+
+ context['revision_id'] = sorted_packages[2].revision_id #original state
+ third_dictized = self.remove_changable_columns(package_dictize(anna1, context))
+
+ second_dictized['name'] = u'annakarenina_changed2'
+ second_dictized['resources'][0]['url'] = u'new_url2'
+ second_dictized['tags'][0]['name'] = u'new_tag'
+ second_dictized['extras'][0]['value'] = u'"new_value"'
+ second_dictized['state'] = 'pending'
+
+ assert second_dictized == third_dictized
+
+ context['revision_id'] = sorted_packages[3].revision_id #original state
+ forth_dictized = self.remove_changable_columns(package_dictize(anna1, context))
+
+ third_dictized['notes'] = 'wee'
+ third_dictized['resources'].insert(2, {u'description': u'',
+ u'format': u'plain text',
+ u'hash': u'',
+ u'position': 2,
+ u'state': u'active',
+ u'url': u'newurl'})
+
+ third_dictized['tags'].insert(1, {'name': u'newnew_tag', 'state': 'active'})
+ third_dictized['extras'].insert(0, {'key': 'david',
+ 'value': u'"new_value"',
+ 'state': u'active'})
+ third_dictized['state'] = 'active'
+
+ pprint(third_dictized)
+ pprint(forth_dictized)
+
+ assert third_dictized == forth_dictized
+
+ def test_14_resource_no_id(self):
context = {"model": model,
"session": model.Session}
@@ -337,15 +674,11 @@
res = model.Session.query(model.Resource).filter_by(url=u'test').one()
-
res_dictized = self.remove_changable_columns(resource_dictize(res, context))
- pprint(res_dictized)
- pprint(new_resource)
-
assert res_dictized == new_resource, res_dictized
- def test_12_api_to_dictize(self):
+ def test_15_api_to_dictize(self):
context = {"model": model,
"session": model.Session}
@@ -410,9 +743,8 @@
pkg = model.Session.query(model.Package).filter_by(name=u'testpkg').one()
package_dictized = self.remove_changable_columns(package_dictize(pkg, context))
- pprint(package_dictized)
- def test_13_group_dictized(self):
+ def test_16_group_dictized(self):
context = {"model": model,
"session": model.Session}
@@ -474,7 +806,7 @@
assert result == expected, pformat(result)
- def test_14_group_apis_to_dict(self):
+ def test_17_group_apis_to_dict(self):
context = {"model": model,
"session": model.Session}
@@ -488,9 +820,6 @@
assert group_api_to_dict(api_group, context) == {'description': u'Great group!',
- 'name': u'testgroup',
- 'packages': [{'id': u'annakarenina'}, {'id': u'warandpeace'}],
- 'title': u'Some Group Title'}, pformat(group_api1_to_dict(api_group, context))
-
-
-
+ 'name': u'testgroup',
+ 'packages': [{'id': u'annakarenina'}, {'id': u'warandpeace'}],
+ 'title': u'Some Group Title'}, pformat(group_api_to_dict(api_group, context))
--- a/ckan/tests/models/test_package.py Thu Jun 23 11:19:31 2011 +0100
+++ b/ckan/tests/models/test_package.py Thu Jun 23 12:30:51 2011 +0100
@@ -38,14 +38,35 @@
rev = model.repo.new_revision()
package = model.Package(name=name)
model.Session.add(package)
+ model.Session.flush()
+ revision_id = model.Session().revision.id
+ timestamp = model.Session().revision.timestamp
model.repo.commit_and_remove()
+ package = model.Package.by_name(name)
+ assert len(package.all_revisions) == 1
+ assert package.all_revisions[0].revision_id == revision_id
+ assert package.all_revisions[0].revision_timestamp == timestamp
+ assert package.all_revisions[0].expired_id is None
+
# change it
rev = model.repo.new_revision()
package = model.Package.by_name(name)
package.title = "wobsnasm"
+ revision_id2 = model.Session().revision.id
+ timestamp2 = model.Session().revision.timestamp
model.repo.commit_and_remove()
+ package = model.Package.by_name(name)
+ assert len(package.all_revisions) == 2
+ assert package.all_revisions[0].revision_id == revision_id2
+ assert package.all_revisions[0].revision_timestamp == timestamp2
+ assert package.all_revisions[0].expired_id is None
+
+ assert package.all_revisions[1].revision_id == revision_id
+ assert package.all_revisions[1].revision_timestamp == timestamp
+ assert package.all_revisions[1].expired_id == revision_id2
+
def test_create_package(self):
package = model.Package.by_name(self.name)
assert package.name == self.name
Repository URL: https://bitbucket.org/okfn/ckan/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the ckan-changes
mailing list