Skip to content

Commit

Permalink
Merge branch 'datapackage'
Browse files Browse the repository at this point in the history
  • Loading branch information
@paulgirard
paulgirard committed Aug 12, 2020
2 parents f274121 + 08a4481 commit d9b21c8
Show file tree
Hide file tree
Showing 32 changed files with 406 additions and 363 deletions.
20 changes: 10 additions & 10 deletions api/controllers/viz.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import modelFlowsPerYear from '../model/flowsPerYear';
import modelCreateLine from '../model/createLine';
import modelTerms from '../model/terms';
import modelNetwork from '../model/country';
import modelNetwork from '../model/partner';
import {mapValues} from 'lodash';

const formatItemsParams = (items) => {
Expand All @@ -27,15 +27,15 @@ const controller = [
// sourceType: '?string',
// productClassification: '?string',
// product: '?string',
// countryClassification: '?string',
// country: '?string',
// partnerClassification: '?string',
// partner: '?string',
// direction: '?string',
// kind: '?string'
// }
// },
action(req, res) {
const payloadFlows = mapValues(req.body, (v, k) => {
if (k === 'product' || k === 'country') {
if (k === 'product' || k === 'partner') {
// separate filters on id from those on name trhough regexp
return formatItemsParams(v);
}
Expand All @@ -54,7 +54,7 @@ const controller = [
method: 'POST',
action(req, res) {
const payload = mapValues(req.body, (v, k) => {
if (k === 'product' || k === 'country') {
if (k === 'product' || k === 'partner') {
// separate filters on id from those on name trhough regexp
return formatItemsParams(v);
}
Expand All @@ -77,15 +77,15 @@ const controller = [
// sourceType: '?string',
// productClassification: '?string',
// product: '?string',
// countryClassification: '?string',
// partnerClassification: '?string',
// kind: '?string',
// dateMin: '?string',
// dateMax: '?string'
// }
// },
action(req, res) {
const payloadNetwork = mapValues(req.body, (v, k) => {
if (k === 'product' || k === 'country') {
if (k === 'product' || k === 'partner') {
// separate filters on id from those on name trhough regexp
return formatItemsParams(v);
}
Expand All @@ -108,8 +108,8 @@ const controller = [
// sourceType: '?string',
// productClassification: '?string',
// product: '?string',
// countryClassification: '?string',
// country: '?string',
// partnerClassification: '?string',
// partner: '?string',
// childClassification: '?string',
// child: '?string',
// direction: '?string',
Expand All @@ -120,7 +120,7 @@ const controller = [
// },
action(req, res) {
const payloadTerms = mapValues(req.body, (v, k) => {
if (k === 'child' || k === 'country') {
if (k === 'child' || k === 'partner') {
// separate filters on id from those on name trhough regexp
return formatItemsParams(v);
}
Expand Down
2 changes: 1 addition & 1 deletion api/model/classification.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ const Model = {

const tree = {
product: makeTree(groupedByModel.product),
country: makeTree(groupedByModel.country)
partner: makeTree(groupedByModel.partner)
};

return callback(null, tree);
Expand Down
46 changes: 23 additions & 23 deletions api/model/createLine.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ const ModelCreateLine = {
kind,
productClassification,
product,
countryClassification,
country
partnerClassification,
partner
} = params;

// Building the query
Expand All @@ -33,17 +33,17 @@ const ModelCreateLine = {
// import export
// define import export edge type filter
let exportImportFilterDirection = ':FROM|:TO';
let exportImportFilterCountry = ':FROM|:TO';
let exportImportFilterPartner = ':FROM|:TO';
if (kind === 'import') {
exportImportFilterDirection = ':TO';
exportImportFilterCountry = ':FROM';
// add a where clause an flow import index to match flows which doesn't have a country or direction link
exportImportFilterPartner = ':FROM';
// add a where clause an flow import index to match flows which doesn't have a partner or direction link
where.and('f.import');
}
else if (kind === 'export') {
exportImportFilterDirection = ':FROM';
exportImportFilterCountry = ':TO';
// add a where clause an flow import index to match flows which doesn't have a country or direction link
exportImportFilterPartner = ':TO';
// add a where clause an flow import index to match flows which doesn't have a partner or direction link
where.and('NOT f.import');
}

Expand All @@ -66,27 +66,27 @@ const ModelCreateLine = {
query.params({productClassification});
}

//-- Do we need to match a country?
if (countryClassification) {
// Adding the country filter in the main query
match.push(`(f:Flow)-[${exportImportFilterCountry}]->(country)`);
where.and(new Expression('country IN countries'));

query.match('(country:Country)<-[:AGGREGATES*1..]-(cci:ClassifiedItem)<-[:HAS]-(cc:Classification)');
const whereCountry = new Expression('cc.id = $countryClassification');
query.params({countryClassification});
if (country) {
const countryFilter = filterItemsByIdsRegexps(country, 'cci');
whereCountry.and(countryFilter.expression);
query.params(countryFilter.params);
//-- Do we need to match a partner?
if (partnerClassification) {
// Adding the partner filter in the main query
match.push(`(f:Flow)-[${exportImportFilterPartner}]->(partner)`);
where.and(new Expression('partner IN partners'));

query.match('(partner:Partner)<-[:AGGREGATES*1..]-(cci:ClassifiedItem)<-[:HAS]-(cc:Classification)');
const wherePartner = new Expression('cc.id = $partnerClassification');
query.params({partnerClassification});
if (partner) {
const partnerFilter = filterItemsByIdsRegexps(partner, 'cci');
wherePartner.and(partnerFilter.expression);
query.params(partnerFilter.params);
}
query.where(whereCountry);
query.where(wherePartner);

if (productClassification) {
query.with('collect(country) AS countries, products');
query.with('collect(partner) AS partners, products');
}
else {
query.with('collect(country) AS countries');
query.with('collect(partner) AS partners');
}
}

Expand Down
50 changes: 25 additions & 25 deletions api/model/flowsPerYear.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,16 @@ const ModelFlowsPerYear = {
direction,
kind,
product,
country
partner
} = params;

let {
productClassification,
countryClassification
partnerClassification
} = params;

let twofoldProduct = false,
twofoldCountry = false;
twofoldPartner = false;

const query = new Query(),
where = new Expression(),
Expand All @@ -55,10 +55,10 @@ const ModelFlowsPerYear = {
twofoldProduct = true;
}
else {
countryClassification = classificationId;
partnerClassification = classificationId;

if (params.countryClassification)
twofoldCountry = true;
if (params.partnerClassification)
twofoldPartner = true;
}

dataType = classificationType;
Expand Down Expand Up @@ -117,23 +117,23 @@ const ModelFlowsPerYear = {
}
}

//-- Do we need to match a country?
if (countryClassification && !twofoldCountry) {
//-- Do we need to match a partner?
if (partnerClassification && !twofoldPartner) {
query.match('(cc)-[:HAS]->(cg:ClassifiedItem)-[:AGGREGATES*0..]->(ci)-[:FROM|:TO]-(f:Flow)');
const whereCountry = new Expression('cc.id = $countryClassification');
query.params({countryClassification});
const wherePartner = new Expression('cc.id = $partnerClassification');
query.params({partnerClassification});

if (country) {
const countryFilter = filterItemsByIdsRegexps(country, 'cg');
if (partner) {
const partnerFilter = filterItemsByIdsRegexps(partner, 'cg');

whereCountry.and(countryFilter.expression);
query.params(countryFilter.params);
wherePartner.and(partnerFilter.expression);
query.params(partnerFilter.params);
}

query.where(whereCountry);
query.where(wherePartner);
withs.add('f');

if (dataType === 'country') {
if (dataType === 'partner') {
query.with([...withs].concat('cg.name as classificationGroupName').join(', '));
withs.add('classificationGroupName');
}
Expand All @@ -142,24 +142,24 @@ const ModelFlowsPerYear = {
}

// NOTE: twofold classification
if (countryClassification && twofoldCountry) {
if (partnerClassification && twofoldPartner) {
query.match('(cc:Classification)-[:HAS]->(cg:ClassifiedItem)-[:AGGREGATES*0..]->(ci)-[:FROM|:TO]-(f:Flow), (ccg:ClassifiedItem)-[:AGGREGATES*0..]->(cg)');
const whereCountry = new Expression('cc.id = $countryClassification');
query.params({countryClassification});
const wherePartner = new Expression('cc.id = $partnerClassification');
query.params({partnerClassification});


if (country) {
const countryFilter = filterItemsByIdsRegexps(country, 'ccg');
if (partner) {
const partnerFilter = filterItemsByIdsRegexps(partner, 'ccg');

whereCountry.and(countryFilter.expression);
query.params(countryFilter.params);
wherePartner.and(partnerFilter.expression);
query.params(partnerFilter.params);
}

query.where(whereCountry);
query.where(wherePartner);

withs.add('f');

if (dataType === 'country') {
if (dataType === 'partner') {
query.with([...withs].concat('cg.name as classificationGroupName').join(', '));
withs.add('classificationGroupName');
}
Expand Down
12 changes: 6 additions & 6 deletions api/model/country.js → api/model/partner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const {Expression, Query} = decypher;
const ModelNetwork = {

/**
* Building the (directions)--(country) network.
* Building the (directions)--(partner) network.
*/
network(classification, params, callback) {

Expand Down Expand Up @@ -50,18 +50,18 @@ const ModelNetwork = {
query.with('collect(product) AS products');
}

// start query from country classification
// start query from partner classification
// define import export edge type filter
let exportImportFilter = ':FROM|:TO';
if (kind === 'import')
exportImportFilter = ':FROM';
else if (kind === 'export')
exportImportFilter = ':TO';
match.push(`(f:Flow)-[${exportImportFilter}]->(:Country)<-[:AGGREGATES*1..]-(cci:ClassifiedItem)<-[:HAS]-(cc:Classification)`);
const whereCountry = new Expression('cc.id = $classification');
match.push(`(f:Flow)-[${exportImportFilter}]->(:Partner)<-[:AGGREGATES*1..]-(cci:ClassifiedItem)<-[:HAS]-(cc:Classification)`);
const wherePartner = new Expression('cc.id = $classification');
query.params({classification});

where.and(whereCountry);
where.and(wherePartner);

//-- Do we need to match a source type?
if (sourceType) {
Expand Down Expand Up @@ -98,7 +98,7 @@ const ModelNetwork = {
if (!where.isEmpty())
query.where(where);

query.return('cci.name as country, f.direction AS direction, count(f) AS count, sum(f.value) AS value');
query.return('cci.name as partner, f.direction AS direction, count(f) AS count, sum(f.value) AS value');

database.cypher(query.build(), function(err, data) {
if (err) return callback(err);
Expand Down
34 changes: 17 additions & 17 deletions api/model/terms.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,40 +21,40 @@ const ModelTerms = {
sourceType,
direction,
kind,
country,
partner,
child,
dateMin,
dateMax,
countryClassification,
partnerClassification,
childClassification
} = params;

const query = new Query(),
where = new Expression(),
match = [];

//-- Do we need to match a country?
if (countryClassification) {
//-- Do we need to match a partner?
if (partnerClassification) {
// define import export edge type filter
let exportImportFilter = ':FROM|:TO';
if (kind === 'import')
exportImportFilter = ':FROM';
else if (kind === 'export')
exportImportFilter = ':TO';
// Adding the country filter in the main query
match.push(`(f:Flow)-[${exportImportFilter}]->(country)`);
where.and(new Expression('country IN countries'));

query.match('(country:Country)<-[:AGGREGATES*1..]-(cci:ClassifiedItem)<-[:HAS]-(cc:Classification)');
const whereCountry = new Expression('cc.id = $countryClassification');
query.params({countryClassification});
if (country) {
const countryFilter = filterItemsByIdsRegexps(country, 'cci');
whereCountry.and(countryFilter.expression);
query.params(countryFilter.params);
// Adding the partner filter in the main query
match.push(`(f:Flow)-[${exportImportFilter}]->(partner)`);
where.and(new Expression('partner IN partners'));

query.match('(partner:Partner)<-[:AGGREGATES*1..]-(cci:ClassifiedItem)<-[:HAS]-(cc:Classification)');
const wherePartner = new Expression('cc.id = $partnerClassification');
query.params({partnerClassification});
if (partner) {
const partnerFilter = filterItemsByIdsRegexps(partner, 'cci');
wherePartner.and(partnerFilter.expression);
query.params(partnerFilter.params);
}
query.where(whereCountry);
query.with('collect(country) AS countries');
query.where(wherePartner);
query.with('collect(partner) AS partners');
}

//-- Do we need to match a product?
Expand Down
12 changes: 6 additions & 6 deletions api/queries/exporter.cypher
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ WITH flow SKIP $offset LIMIT $limit

OPTIONAL MATCH (flow)-[:OF]->(product:Product)
OPTIONAL MATCH (flow)-[transcription:TRANSCRIBED_FROM]->(source:Source)
OPTIONAL MATCH (flow)-[:FROM|:TO]-(country:Country)
OPTIONAL MATCH (flow)-[:FROM|:TO]-(partner:Partner)
OPTIONAL MATCH (flow)-[:FROM|:TO]-(direction:Direction)
OPTIONAL MATCH (flow)-[:FROM|:TO]-(office:Office)
OPTIONAL MATCH (flow)-[:TRANSCRIBED_BY]->(operator:Operator)
Expand All @@ -21,7 +21,7 @@ RETURN
operator.name AS operator,
direction.name AS direction,
office.name AS office,
country.name AS country,
partner.name AS partner,
origin.name AS origin

// name: classifications
Expand All @@ -37,11 +37,11 @@ RETURN c AS classification, p.slug AS parent
MATCH (p:Product)
RETURN p.name AS product, ["toflit18"] + [(p)-[:TRANSCRIBED_FROM]->(source)|source.name] AS sources

// name: countries
// Retrieving every source country.
// name: partners
// Retrieving every source partner.
//------------------------------------------------------------------------------
MATCH (c:Country)
RETURN c.name AS country
MATCH (c:Partner)
RETURN c.name AS partner

// name: classifiedItemsToSource
// Retrieving groups from a classification and mapping them to the sources.
Expand Down
Loading

0 comments on commit d9b21c8

Please sign in to comment.