[wdmmg-dev] Denormalization and model.mongo.Base.to_ref_dict()
Carsten Senger
senger at rehfisch.de
Thu May 12 17:22:49 UTC 2011
We use Base.to_ref_dict() to make a copy of the informations of a model
(dataset, entities, classifiers) and save it on every entry. to_ref_dict()
generates a dictionary with all data of the other model including views. I
added a sample to the end of the mail.
I've written code to do the aggregations for the new api's which saves
information on the dataset document after the views where applied, and that
broke a test where to_ref_dict() was used to query an entry. I fixed it
here: <https://bitbucket.org/okfn/wdmmg/changeset/2547a562f441> and did not
find it used that way in real code. This makes me wonder if we can reduce
the amount of data we generate in to_ref_dict() to '_id', 'name', 'label'
and 'ref' (maybe 'description' too). This reduces the number of cases where
we would have to update the entries with new values if e.g. an entity
changes, and it reduces the amount of data stored, indexed and
serialized/deserialized. Looking through the code I found no place where we
would need more informaion, but maybe someone know more.
Btw. we don't have a strategy to update denormalized values atm, e.g. if a
dataset import updates an entity, existing entries from other datasets are
not updated.
..Carsten
{
"_id" : ObjectId("4d8aaf6285b2e75fbf71344f"),
"actiontype" : "08 Research",
"amount" : 3490,
"article" : {
"name" : "4-section-3-08-22-04",
"parent" : {
"$ref" : "classifier",
"$id" : ObjectId("4d853244d8d8bb735e97d1c6")
},
"views" : {
"default" : {
"breakdown" : "item",
"label" : "Spending by Item",
"dimension" : "article",
"filters" : "{\"flow\": \"spending\", \"projection\": true,
\"commitment\": true}",
"dataset" : "eu"
},
"fts_beneficiary" : {
"breakdown" : "to",
"label" : "FTS Beneficiaries of Commitments",
"dimension" : "article",
"filters" : "{}",
"dataset" : "fts"
},
"fts_subjects" : {
"breakdown" : "grant_subject",
"label" : "FTS Descriptions of Commitments",
"dimension" : "article",
"filters" : "{}",
"dataset" : "fts"
},
"fts_ms" : {
"breakdown" : "country",
"label" : "FTS Member States of Beneficiaries",
"dimension" : "article",
"filters" : "{}",
"dataset" : "fts"
}
},
"taxonomy" : "eu",
"legal_basis" : "",
"label" : "Appropriations accruing from contributions from (non-European
Economic Area) third parties to research and technological development",
"level_name" : "article",
"_id" : ObjectId("4d85325cd8d8bb735e97dc4d"),
"ref" : {
"$ref" : "classifier",
"$id" : ObjectId("4d85325cd8d8bb735e97dc4d")
},
"description" : "<p>\n<em>Former Article 08 21 04</em>\n</p>\n\n<p>This
appropriation will cover expenditure corresponding to revenue giving rise
to the making available of additional appropriations from (non-European
Economic Area) third parties or third countries participating in projects
in the field of research and technological development.</p>\n\n<p>In
accordance with Article 18 of the Financial Regulation, any revenue entered
in Article 6 0 1 of the statement of revenue may give rise to the
provision of additional appropriations.</p>\n\n<p>Any revenue from the
contribution from candidate countries for participation in Community
programmes, entered in Item 6 0 3 1 of the statement of revenue, may give
rise to the provision of additional appropriations in accordance with
Article 18(1)(d) of the Financial Regulation.</p>\n\n<p>Any revenue from
the contribution from outside bodies for participation in Community
programmes, entered in Item 6 0 3 3 of the statement of revenue, may give
rise to the provision of additional appropriations in accordance with
Article 18(1)(d) of the Financial Regulation.</p>\n\n"
},
"budget_code" : "08.22.04",
"budget_item" : "Appropriations accruing from contributions from
(non-European Economic Area) third parties to research and technological
development",
"chapter" : {
"name" : "4-section-3-08-22",
"parent" : {
"$ref" : "classifier",
"$id" : ObjectId("4d853244d8d8bb735e97d1db")
},
"views" : {
"default" : {
"breakdown" : "article",
"label" : "Spending by Article",
"dimension" : "chapter",
"filters" : "{\"flow\": \"spending\", \"projection\": true,
\"commitment\": true}",
"dataset" : "eu"
}
},
"taxonomy" : "eu",
"legal_basis" : "",
"label" : "Completion of previous framework programmes and other
activities",
"level_name" : "chapter",
"_id" : ObjectId("4d853249d8d8bb735e97d62b"),
"ref" : {
"$ref" : "classifier",
"$id" : ObjectId("4d853249d8d8bb735e97d62b")
},
"description" : ""
},
"classifier_ids" : [
ObjectId("4d853244d8d8bb735e97d1db"),
ObjectId("4d85325cd8d8bb735e97dc4d"),
ObjectId("4d853249d8d8bb735e97d62b")
],
"classifiers" : [
[
"4-section-3-08-22",
"eu"
],
[
"4-section-3-08-22-04",
"eu"
],
[
"4-section-3-08",
"eu"
]
],
"cofinancing_rate" : " 100,00 %",
"cofinancing_rate_pct" : 100,
"currency" : "EUR",
"dataset" : {
"time_axis" : "time.from.year",
"description" : "Financial commitments by the EuropeanCommission",
"views" : {
"default" : {
"breakdown" : "from",
"label" : "Commitments per Directorate-General",
"dimension" : "dataset",
"filters" : "{}",
"dataset" : "fts"
},
"actiontype" : {
"breakdown" : "actiontype",
"label" : "Commitments per Action Type",
"dimension" : "dataset",
"filters" : "{}",
"dataset" : "fts"
}
},
"label" : "EC Financial Transparency System",
"currency" : "EUR",
"_id" : ObjectId("4d8aae8c85b2e75fbf6f8061"),
"ref" : {
"$ref" : "dataset",
"$id" : ObjectId("4d8aae8c85b2e75fbf6f8061")
},
"name" : "fts"
},
"from" : {
"description" : "",
"views" : {
"default" : {
"breakdown" : "article",
"label" : "Commitments per Budget Article",
"dimension" : "from",
"filters" : "{}",
"dataset" : "fts"
},
"actiontype" : {
"breakdown" : "actiontype",
"label" : "Commitments per Action Type",
"dimension" : "from",
"filters" : "{}",
"dataset" : "fts"
}
},
"label" : "Research",
"fts_department" : true,
"_id" : ObjectId("4d89c4d7bccc4fa374e85911"),
"ref" : {
"$ref" : "entity",
"$id" : ObjectId("4d89c4d7bccc4fa374e85911")
},
"name" : "ec-research"
},
"grant_subject" : "",
"is_aggregate" : false,
"name" :
"fts-si2-494925-1-ec-research-provisional-commitment-beneficiary-not-identified-at-this-stage-None",
"position_key" : "SI2.494925.1",
"source_file" :
"/srv/webapps/wdmmg/wdmmg/pylons_data/getdata/fts/export_2008_en.xml",
"time" : {
"to" : {
"month" : "200812",
"parsed" : ISODate("2008-12-31T00:00:00Z"),
"day" : "20081231",
"year" : "2008"
},
"unparsed" : "2008",
"from" : {
"month" : "200801",
"parsed" : ISODate("2008-01-01T00:00:00Z"),
"day" : "20080101",
"year" : "2008"
}
},
"title" : {
"name" : "4-section-3-08",
"parent" : {
"$ref" : "classifier",
"$id" : ObjectId("4d853244d8d8bb735e97d1da")
},
"views" : {
"default" : {
"breakdown" : "chapter",
"label" : "Spending by Chapter",
"dimension" : "title",
"filters" : "{\"flow\": \"spending\", \"projection\": true,
\"commitment\": true}",
"dataset" : "eu"
}
},
"taxonomy" : "eu",
"legal_basis" : "",
"label" : "Research",
"level_name" : "title",
"_id" : ObjectId("4d853244d8d8bb735e97d1db"),
"ref" : {
"$ref" : "classifier",
"$id" : ObjectId("4d853244d8d8bb735e97d1db")
},
"description" : ""
},
"to" : {
"fts_beneficiary" : true,
"city" : "",
"geozone" : "",
"post_code" : "",
"description" : "",
"views" : {
"default" : {
"breakdown" : "article",
"label" : "FTS Commitments Received (by Article)",
"dimension" : "to",
"filters" : "{}",
"dataset" : "fts"
}
},
"country" : "",
"label" : "Provisional commitment - beneficiary not identified at this
stage",
"coordinator" : false,
"address" : "",
"_id" : ObjectId("4d89f975bccc4fa374e94a81"),
"ref" : {
"$ref" : "entity",
"$id" : ObjectId("4d89f975bccc4fa374e94a81")
},
"name" : "provisional-commitment-beneficiary-not-identified-at-this-stage"
},
"total" : 3490
}
More information about the openspending-dev
mailing list