[ckan-changes] commit/ckanext-dgu: dread: [ons]: Improved NS filter to check designation.

Mon Nov 14 16:49:47 UTC 2011

1 new commit in ckanext-dgu:


https://bitbucket.org/okfn/ckanext-dgu/changeset/ef03bd82c3cf/
changeset:   ef03bd82c3cf
user:        dread
date:        2011-11-14 17:44:48
summary:     [ons]: Improved NS filter to check designation.
affected #:  2 files

diff -r 9269251e1c983cc3986500ee34c950b85b7ecc03 -r ef03bd82c3cfaf0bcbec40a412e904c9d408f71c ckanext/dgu/bin/national_statistic_filter.py

--- a/ckanext/dgu/bin/national_statistic_filter.py
+++ b/ckanext/dgu/bin/national_statistic_filter.py
@@ -49,18 +49,17 @@
                 is_ns = False
             if is_ns:
                 if not pkg['extras'].get('import_source', '').startswith('ONS'):
-                    save_result('NS but not ONS - change', pkg)
-                    if not self.dry_run:
-                        pkg['extras']['national_statistic'] = 'no'
-                        try:
-                            self.client.package_entity_put(pkg)
-                        except CkanApiError, e:
-                            log.error('Could not update package %r: %r' % (pkg['name'], e.args))
-                            if not self.force:
-                                raise
-                        log.info('Changed ok')
+                    save_result('NS but not from ONS Pub Hub - change', pkg)
+                    self.clear_ns_flag(pkg)
                 else:
-                    save_result('NS and ONS', pkg)
+                    notes = pkg['extras'].get('notes', '')
+                    if 'Designation' in notes:
+                        if 'Designation: National Statistic' in notes:
+                            save_result('NS, from ONS Pub Hub, Designated NS', pkg)
+                        else:
+                            save_result('NS, from ONS Pub Hub but not Designated NS - change', pkg)
+                            self.clear_ns_flag(pkg)
+                            
             else:
                 save_result('Not NS', pkg)
 
@@ -68,6 +67,17 @@
         for reason, pkgs in results.items():
             log.info('  %i %s: %r', len(pkgs), reason, pkgs[:5])
 
+    def clear_ns_flag(self, pkg):
+        if not self.dry_run:
+            pkg['extras']['national_statistic'] = 'no'
+            try:
+                self.client.package_entity_put(pkg)
+            except CkanApiError, e:
+                log.error('Could not update package %r: %r' % (pkg['name'], e.args))
+                if not self.force:
+                    raise
+            log.info('Changed ok')
+
 class Command(MassChangerCommand):
     '''Looks through all packages and if one is marked national statistic, but is
 not from the hub, then unmark it.
@@ -81,5 +91,8 @@
                              force=self.options.force)
         ns_filter.filter()
 
+    def add_options(self):
+        super(Command, self).add_options()
+
 if __name__ == '__main__':
     Command().command()


diff -r 9269251e1c983cc3986500ee34c950b85b7ecc03 -r ef03bd82c3cfaf0bcbec40a412e904c9d408f71c ckanext/dgu/tests/bin/test_national_statistic_filter.py
--- a/ckanext/dgu/tests/bin/test_national_statistic_filter.py
+++ b/ckanext/dgu/tests/bin/test_national_statistic_filter.py
@@ -21,6 +21,14 @@
             {'name': "ons_pkg",
              "extras": {
                  "import_source": "ONS-ons_data_7_days_to_2011-05-10",
+                 "notes": "<p>Designation: National Statistics\n</p>",
+                 "national_statistic": "yes",
+                 }
+             },
+            {'name': "ons_but_not_ns",
+             "extras": {
+                 "import_source": "ONS-ons_data_7_days_to_2011-05-10",
+                 "notes": "<p>Designation: Excellent Statistics\n</p>",
                  "national_statistic": "yes",
                  }
              },
@@ -58,7 +66,7 @@
                  "mandate": "", "date_update_future": "",
                  "update_frequency": "monthly",
                  "categories": "Government"}
-             }
+             },            
             ]
         CreateTestData.create_arbitrary(self.pkgs,
                                         extra_user_names=[username])
@@ -76,12 +84,18 @@
         ns_filter = NSFilter(self.testclient, dry_run=False, force=False)
         ns_filter.filter()
 
-        PackageDictUtil.assert_subset(model.Package.by_name(u'ons_pkg').as_dict(), self.pkgs[0])
-        PackageDictUtil.assert_subset(model.Package.by_name(u'not_ns_or_ons').as_dict(), self.pkgs[1])
-        PackageDictUtil.assert_subset(model.Package.by_name(u'not_ns').as_dict(), self.pkgs[2])
-
-        pkg = model.Package.by_name(u'local-authority-spend-over-500-london-borough-of-hackney')
-        expected_pkg = copy.deepcopy(self.pkgs[3])
-        expected_pkg['extras']['national_statistic'] = 'no'
-        PackageDictUtil.assert_subset(pkg.as_dict(), expected_pkg)
+        def assert_pkg_stayed_the_same(package_name, pkg_dict):
+            pkg = model.Package.by_name(unicode(package_name))
+            PackageDictUtil.assert_subset(pkg.as_dict(), pkg_dict)
+            
+        def assert_pkg_filtered(package_name, pkg_dict):
+            pkg = model.Package.by_name(unicode(package_name))
+            expected_pkg = copy.deepcopy(pkg_dict)
+            expected_pkg['extras']['national_statistic'] = 'no'
+            PackageDictUtil.assert_subset(pkg.as_dict(), expected_pkg)
         
+        assert_pkg_stayed_the_same('ons_pkg', self.pkgs[0])
+        assert_pkg_filtered('ons_but_not_ns', self.pkgs[1])
+        assert_pkg_stayed_the_same('not_ns_or_ons', self.pkgs[2])
+        assert_pkg_stayed_the_same('not_ns', self.pkgs[3])
+        assert_pkg_filtered('local-authority-spend-over-500-london-borough-of-hackney', self.pkgs[4])

Repository URL: https://bitbucket.org/okfn/ckanext-dgu/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.


