[ckan-dev] Harvester - Problem

Michael Reichart michael.reichart at gmail.com
Thu Jan 17 08:21:47 UTC 2013


Maybe it helps to know, that we are using the organizations - extension


Thanks
Michael


2013/1/17 Michael Reichart <michael.reichart at gmail.com>

> Hi,
>
> datasets created and updated by the webinterface do not loose their
> groups. I haven't tested the direct api-calls yet.
> I can't share our full extension, but i've attached the full code of the
> IDatasetForm
> We're using a custom CKAN-Harvester (which only does some mapping to our
> custom fields) based on ckanext-harvest from beginning of July 2012 could
> be Version 1.6.12??
>
> From my understanding, it shouldn't be necessary to call our custom schema
> to import from another CKAN as the data-objects are already in the right
> form.
>
> Thanks!
> Michael
>
> class DgvatForm(SingletonPlugin):
>     implements(IRoutes)
>     implements(IConfigurer)
>     implements(IGenshiStreamFilter)
>     implements(IDatasetForm)
>
>     log.fatal("Enter: %s" % __name__)
>     def package_form(self):
>         return 'package/datagvat_organization_form.html'
>
>
>     def before_map(self, map):
>         map.connect('/error/{action}',
> controller='ckanext.dgvat_por.controllers.data_gv_at:DgvatErrorController')
>         map.connect('/error/{action}/{id}',
> controller='ckanext.dgvat_por.controllers.data_gv_at:DgvatErrorController')
>
>
>         map.connect('/dataset/new',
> controller='ckanext.dgvat_por.controllers.data_gv_at:DgvatPackageController',
> action='new')
>         map.connect('/dataset/edit/{id}',
> controller='ckanext.dgvat_por.controllers.data_gv_at:DgvatPackageController',
> action='edit')
>         map.connect('/dataset/{id}.{format}',
> controller='ckanext.dgvat_por.controllers.data_gv_at:DgvatPackageController',
> action='read')
>         map.connect('/dataset/{id}',
> controller='ckanext.dgvat_por.controllers.data_gv_at:DgvatPackageController',
> action='read')
>         map.connect('/dataset/{id}/resource/{resource_id}',
> controller='ckanext.dgvat_por.controllers.data_gv_at:DgvatPackageController',
> action='resource_read')
>         map.connect('/dataset/{id}/resource/{resource_id}/embed',
> controller='ckanext.dgvat_por.controllers.data_gv_at:DgvatPackageController',
> action='resource_embedded_dataviewer')
>         map.connect('/dataset/editresources/{id}',
> controller='ckanext.dgvat_por.controllers.data_gv_at:DgvatPackageController',
> action='editresources')
>         map.connect('/user/edit/{id:.*}',
> controller='ckanext.dgvat_por.controllers.dgvat_user:DgvatUserController',
> action='edit')
>         #m.connect('/user/reset/{id:.*}', action='perform_reset')
>         map.connect('/user/register',
> controller='ckanext.dgvat_por.controllers.dgvat_user:DgvatUserController',
> action='register')
>         map.connect('/user/login',
> controller='ckanext.dgvat_por.controllers.dgvat_user:DgvatUserController',
> action='login')
>         map.connect('/user/_logout', '/user/logout',
> controller='ckanext.dgvat_por.controllers.dgvat_user:DgvatUserController',
> action='logout')
>         map.connect('/user/logged_in',
> controller='ckanext.dgvat_por.controllers.dgvat_user:DgvatUserController',
> action='logged_in')
>         map.connect('/user/logged_out',
> controller='ckanext.dgvat_por.controllers.dgvat_user:DgvatUserController',
> action='logged_out')
>         map.connect('/user/logged_out_redirect',
> controller='ckanext.dgvat_por.controllers.dgvat_user:DgvatUserController',
> action='logged_out_page')
>         #map.connect('/user/reset', action='request_reset')
>         #m.connect('/user/me', action='me')
>         map.connect('/user/set_lang/{lang}',
> controller='ckanext.dgvat_por.controllers.dgvat_user:DgvatUserController',
> action='set_lang')
>         #m.connect('/user/{id:.*}', action='read')
>         map.connect('/user',
> controller='ckanext.dgvat_por.controllers.dgvat_user:DgvatUserController',
> action='index')
>
>         map.connect('home', '/',
> controller='ckanext.dgvat_por.controllers.dgvat_cockpit:DgvatCockpitController',
> action='search')
>         map.connect('about', '/about',
> controller='ckanext.dgvat_por.controllers.dgvat_cockpit:DgvatCockpitController',
> action='about')
>         map.connect('terms', '/terms',
> controller='ckanext.dgvat_por.controllers.dgvat_cockpit:DgvatCockpitController',
> action='terms')
>         map.connect('help', '/help',
> controller='ckanext.dgvat_por.controllers.dgvat_cockpit:DgvatCockpitController',
> action='help')
>         map.connect('register_api', '/register_api',
> controller='ckanext.dgvat_por.controllers.dgvat_cockpit:DgvatCockpitController',
> action='register')
>
>         #log.fatal("==================================> %s" % map)
>         return map
>
>
>     def after_map(self, map):
>         #log.fatal("==================================> %s" % map)
>         return map
>
>
>     def update_config(self, config):
>         log.debug("update_config")
>         config['package_form'] = 'data_gv_at'
>         configure_template_directory(config, 'templates')
>         configure_public_directory(config, 'public')
>
>
>     def filter(self, stream):
>
>         from pylons import request, tmpl_context as c
>         routes = request.environ.get('pylons.routes_dict')
>         log.debug(routes)
>         if routes and \
>                routes.get('controller') == 'package' and \
>                routes.get('action') == 'read' and c.pkg.id:
>
>             # Add dataset id to the UI
>             stream = stream_filters.package_id_filter(stream, c.pkg)
>         return stream
>
>
>     def setup_template_variables(self, context, data_dict):
>         log.debug("setup_template_variables")
>
>     def package_types(self):
>         return ['dataset']
>
>     def is_fallback(self):
>         return True
>
>     def form_to_db_schema(self):
>         log.fatal("Enter form to db!!!")
>
>         schema = {
>                   'title': [if_empty_same_as("name"), unicode],
>                   'name': [not_empty, unicode, val.name_validator,
> val.package_name_validator],
>                   'license_id': [ignore_missing, unicode],
>                   'maintainer': [ignore_missing, unicode],
>                   'schema_name': [ignore_missing, unicode,
> convert_to_extras],
>                   'maintainer_link': [ignore_missing, unicode,
> convert_to_extras],
>                   'schema_language': [ignore_missing,
> unicode,convert_to_extras],
>                   'schema_characterset': [ignore_missing,
> unicode,convert_to_extras],
>                   'date_released': [ignore_missing, unicode,
> convert_to_extras],
>                   'begin_datetime': [ignore_missing, unicode,
> convert_to_extras],
>                   'end_datetime': [ignore_missing, unicode,
> convert_to_extras],
>                   'metadata_linkage': [ignore_missing, unicode,
> convert_to_extras],
>                   'attribute_description': [ignore_missing, unicode,
> convert_to_extras],
>                   'publisher': [ignore_missing, unicode,
> convert_to_extras],
>                   'geographic_toponym': [ignore_missing, unicode,
> convert_to_extras],
>                   'geographic_bbox': [ignore_missing, unicode,
> convert_to_extras],
>                   'lineage_quality': [ignore_missing, unicode,
> convert_to_extras],
>                   'en_title_and_desc': [ignore_missing, unicode,
> convert_to_extras],
>                   'license_citation':[ignore_missing, unicode,
> convert_to_extras],
>                   'metadata_identifier':[ignore_missing, unicode,
> convert_to_extras],
>                   'metadata_modified':[ignore_missing, unicode,
> convert_to_extras],
>                   'date_updated':[ignore_missing, unicode,
> convert_to_extras],
>                   'url': [ignore_missing, unicode],
>                   'resources': default_schema.default_resource_schema(),
>                   'state': [val.ignore_not_admin, ignore_missing],
>                   'log_message': [unicode, val.no_http],
>                   '__extras': [ignore],
>                   'revision_id': [ignore],
>                   'update_frequency': [ignore_missing, unicode,
> convert_to_extras],
>                   'categorization': [ignore_missing, unicode,
> self.add_to_extras],
>                   'notes': [ignore_missing, unicode],
>                   'tag_string': [ignore_missing, val.tag_string_convert],
>                   'groups': {
>                              'id': [ignore_missing, unicode],
>                              'capacity': [ignore_missing, unicode],
>                              '__extras': [ignore],
>                             },
>                   }
>
>         schema['resources'].update({
>                     'language':[ignore_missing],
>                     'characterset':[ignore_missing]
>                        })
>
>
>         return schema
>
>     def add_to_extras(self, key, data, errors, context):
>         # get current number of extras
>         extras = data.get(('extras',), [])
>         if not extras:
>             data[('extras',)] = extras
>         #extras.append({'key': key[-1], 'value': data[key]})
>         #log.fatal("add to extras: %s, %s" % (key, data[key]))
>         #log.fatal("extra_number: %s" % extra_number)
>         #data[('extras', extra_number, 'key')] = key[0]
>         cats =""
>         allCat=[]
>         for cur in data[key]:
>             if cur != '[' and cur!=']':
>                 cats += cur
>         for cat_desc, id in c.categorization:
>             if id in cats:
>                 allCat.append(id)
>         log.fatal("added categorizations:id %s" % allCat)
>         log.fatal("cats, key: %s,%s" % (cats, data[key]))
>         log.fatal("is list: %s" % isinstance(data[key], list))
>         #data[('extras', extra_number, 'value')] = allCat
>         extras.append({'key': key[-1], 'value': allCat})
>
>
>
>     def db_to_form_schema(self):
>         log.debug("Enter db to form")
>         #c.categorization = gv.categorization
>         #c.update_frequency = gv.update_frequency
>         schema = logic.schema.package_form_schema()
>         schema.update({
>             'schema_name': [convert_from_extras, ignore_missing, unicode],
>             'schema_language': [convert_from_extras, ignore_missing,
> unicode],
>             'maintainer_link': [convert_from_extras, ignore_missing,
> unicode],
>             'date_released': [convert_from_extras, ignore_missing,
> unicode],
>             'begin_datetime': [convert_from_extras, ignore_missing,
> unicode],
>             'end_datetime': [convert_from_extras, ignore_missing, unicode],
>             'metadata_linkage': [convert_from_extras, ignore_missing,
> unicode],
>             'attribute_description': [convert_from_extras, ignore_missing,
> unicode],
>             'publisher': [convert_from_extras, ignore_missing, unicode],
>             'geographic_toponym': [convert_from_extras, ignore_missing,
> unicode],
>             'geographic_bbox': [convert_from_extras, ignore_missing,
> unicode],
>             'lineage_quality': [convert_from_extras, ignore_missing,
> unicode],
>             'en_title_and_desc': [convert_from_extras, ignore_missing,
> unicode],
>             'license_citation':[convert_from_extras, ignore_missing,
> unicode],
>             'metadata_identifier':[convert_from_extras, ignore_missing,
> unicode],
>             'metadata_modified':[convert_from_extras, ignore_missing,
> unicode],
>             'date_updated':[convert_from_extras, ignore_missing, unicode],
>             'resources': default_schema.default_resource_schema(),
>             'categorization': [convert_from_extras, ignore_missing],
>             'update_frequency': [convert_from_extras, ignore_missing],
>             'extras': {
>                 'key': [],
>                 'value': [],
>                 '__extras': [keep_extras]
>             },
>             'tags': {
>                 '__extras': [keep_extras]
>             },
>             'mandate': [convert_from_extras, ignore_missing],
>             'national_statistic': [convert_from_extras, ignore_missing],
>             '__extras': [keep_extras],
>         })
>
>         schema['groups'].update({
>             'name': [not_empty, unicode],
>             'title': [ignore_missing],
>             'capacity': [ignore_missing, unicode]
>         })
>
>         schema['resources'].update({
>             'created': [ignore_missing],
>             'position': [not_empty],
>             'last_modified': [ignore_missing],
>             'cache_last_updated': [ignore_missing],
>             'webstore_last_updated': [ignore_missing],
>             'language':[ignore_missing],
>             'characterset':[ignore_missing]
>         })
>
>         return schema
>
>     def read_template(self):
>         return 'package/read.html'
>
>
>     def check_data_dict(self, schema=None):
>         return
>
>     def new_template(self):
>         return 'package/new.html'
>
>     def comments_template(self):
>         return 'package/comments.html'
>
>     def search_template(self):
>         return 'package/search.html'
>
>     def read_template(self):
>         return 'package/read.html'
>
>     def history_template(self):
>         return 'package/history.html'
>
>     def package_form(self):
>         return 'package/datagvat_organization_form.html'
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.okfn.org/pipermail/ckan-dev/attachments/20130117/5a2743c6/attachment-0001.html>


More information about the ckan-dev mailing list