Skip to content

Commit 5e041c4

Browse files
underbluewatersunderbluewaters
authored
Metadata file storage (#820)
* WIP - Add xml metadata file storage and handling as a data upload output * WIP * WIP * Remove console.log * Move endpoint check to function body * Commit db changes * Fix for db migration issue * Fixing missing R2_TILES_BUCKET
1 parent f977f36 commit 5e041c4

26 files changed

+826
-128
lines changed

‎.github/workflows/deploy.yml

+1
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ jobs:
143143
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
144144
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
145145
R2_FILE_UPLOADS_BUCKET: ${{ secrets.R2_FILE_UPLOADS_BUCKET }}
146+
R2_TILES_BUCKET: ${{ secrets.R2_TILES_BUCKET }}
146147
CLOUDFLARE_IMAGES_ACCOUNT: ${{ secrets.CLOUDFLARE_IMAGES_ACCOUNT }}
147148
CLOUDFLARE_IMAGES_TOKEN: ${{ secrets.CLOUDFLARE_IMAGES_TOKEN }}
148149
CLOUDFLARE_IMAGES_ACCOUNT_HASH: ${{ secrets.CLOUDFLARE_IMAGES_ACCOUNT_HASH }}

‎packages/api/generated-schema-clean.gql

+4
Original file line numberDiff line numberDiff line change
@@ -3390,6 +3390,7 @@ enum DataUploadOutputType {
33903390
GEO_TIFF
33913391
PMTILES
33923392
PNG
3393+
XMLMETADATA
33933394
ZIPPED_SHAPEFILE
33943395
}
33953396

@@ -9574,6 +9575,7 @@ type Mutation {
95749575
"""
95759576
input: UpdateTableOfContentsItemChildrenInput!
95769577
): UpdateTableOfContentsItemChildrenPayload
9578+
updateTocMetadataFromXML(filename: String, id: Int!, xmlMetadata: String!): TableOfContentsItem!
95779579

95789580
"""Updates a single `Topic` using a unique key and a patch."""
95799581
updateTopic(
@@ -14329,6 +14331,8 @@ type TableOfContentsItem implements Node {
1432914331
DraftJS compatible representation of text content to display when a user requests layer metadata. Not valid for Folders
1433014332
"""
1433114333
metadata: JSON
14334+
metadataFormat: String
14335+
metadataXml: DataUploadOutput
1433214336

1433314337
"""
1433414338
A globally unique identifier. Can be used in various places throughout the system to identify this single value.

‎packages/api/generated-schema.gql

+4-1
Original file line numberDiff line numberDiff line change
@@ -3390,6 +3390,7 @@ enum DataUploadOutputType {
33903390
GEO_TIFF
33913391
PMTILES
33923392
PNG
3393+
XMLMETADATA
33933394
ZIPPED_SHAPEFILE
33943395
}
33953396

@@ -9574,7 +9575,7 @@ type Mutation {
95749575
"""
95759576
input: UpdateTableOfContentsItemChildrenInput!
95769577
): UpdateTableOfContentsItemChildrenPayload
9577-
updateTocMetadataFromXML(id: Int!, xmlMetadata: String!): TableOfContentsItem!
9578+
updateTocMetadataFromXML(filename: String, id: Int!, xmlMetadata: String!): TableOfContentsItem!
95789579

95799580
"""Updates a single `Topic` using a unique key and a patch."""
95809581
updateTopic(
@@ -14330,6 +14331,8 @@ type TableOfContentsItem implements Node {
1433014331
DraftJS compatible representation of text content to display when a user requests layer metadata. Not valid for Folders
1433114332
"""
1433214333
metadata: JSON
14334+
metadataFormat: String
14335+
metadataXml: DataUploadOutput
1433314336

1433414337
"""
1433514338
A globally unique identifier. Can be used in various places throughout the system to identify this single value.

‎packages/api/migrations/committed/000335.sql

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
--! Previous: sha1:b29221121218a7daf08c79f2e2580f66236bfdfc
2+
--! Hash: sha1:fc2579e6e5275d22de4e81f534d1666a014721f8
3+
4+
-- Enter migration here
5+
alter type data_upload_output_type add value if not exists 'XMLMetadata';
6+
7+
drop function if exists table_of_contents_items_metadata_xml;
8+
create or replace function table_of_contents_items_metadata_xml(item table_of_contents_items)
9+
returns data_upload_outputs
10+
language plpgsql
11+
stable
12+
security definer
13+
as $$
14+
declare
15+
ds_id int;
16+
data_upload_output data_upload_outputs;
17+
begin
18+
select data_source_id into ds_id from data_layers where id = item.data_layer_id;
19+
if ds_id is null then
20+
return null;
21+
end if;
22+
select * into data_upload_output from data_upload_outputs where data_source_id = ds_id and type = 'XMLMetadata';
23+
return data_upload_output;
24+
end;
25+
$$;
26+
27+
grant execute on function table_of_contents_items_metadata_xml(table_of_contents_items) to anon;
28+
29+
drop function if exists create_metadata_xml_output;
30+
create or replace function create_metadata_xml_output(data_source_id int, url text, remote text, size bigint, filename text, metadata_type text)
31+
returns data_upload_outputs
32+
language plpgsql
33+
security definer
34+
as $$
35+
declare
36+
source_exists boolean;
37+
output_id int;
38+
output data_upload_outputs;
39+
original_fname text;
40+
pid int;
41+
begin
42+
-- first, check if data_source even exists
43+
select exists(select 1 from data_sources where id = data_source_id) into source_exists;
44+
if source_exists = false then
45+
raise exception 'Data source does not exist';
46+
end if;
47+
48+
select
49+
original_filename,
50+
project_id
51+
into original_fname, pid
52+
from data_upload_outputs
53+
where
54+
data_upload_outputs.data_source_id = create_metadata_xml_output.data_source_id;
55+
if session_is_admin(pid) = false then
56+
raise exception 'Only admins can create metadata xml outputs';
57+
end if;
58+
-- delete existing metadata xml output
59+
delete from
60+
data_upload_outputs
61+
where
62+
data_upload_outputs.data_source_id = create_metadata_xml_output.data_source_id and
63+
type = 'XMLMetadata';
64+
insert into data_upload_outputs (
65+
data_source_id,
66+
project_id,
67+
type,
68+
url,
69+
remote,
70+
size,
71+
filename,
72+
original_filename
73+
) values (
74+
create_metadata_xml_output.data_source_id,
75+
pid,
76+
'XMLMetadata',
77+
url,
78+
remote,
79+
size,
80+
filename,
81+
original_fname
82+
) returning * into output;
83+
UPDATE data_sources
84+
SET geostats = jsonb_set(
85+
geostats,
86+
'{layers,0,metadata,type}',
87+
metadata_type::jsonb,
88+
true
89+
)
90+
WHERE data_sources.id = id and
91+
jsonb_typeof(geostats->'layers'->0->'metadata'->'type') IS NOT NULL;
92+
return output;
93+
end;
94+
$$;
95+
96+
grant execute on function create_metadata_xml_output to seasketch_user;
97+
98+
comment on function create_metadata_xml_output is '@omit';
99+
100+
drop function if exists table_of_contents_items_metadata_format;
101+
create or replace function table_of_contents_items_metadata_format(item table_of_contents_items)
102+
returns text
103+
language plpgsql
104+
stable
105+
security definer
106+
as $$
107+
declare
108+
ds_id int;
109+
metadata_type text;
110+
begin
111+
select data_source_id into ds_id from data_layers where id = item.data_layer_id;
112+
SELECT geostats->'layers'->0->'metadata'->>'type' into metadata_type
113+
FROM data_sources
114+
WHERE jsonb_typeof(geostats->'layers'->0->'metadata'->'type') IS NOT NULL;
115+
return metadata_type;
116+
end;
117+
$$;
118+
119+
grant execute on function table_of_contents_items_metadata_format(table_of_contents_items) to anon;

‎packages/api/process-env.d.ts

+1
Original file line numberDiff line numberDiff line change
@@ -49,5 +49,6 @@ declare namespace NodeJS {
4949
CLOUDFLARE_ACCOUNT_TAG: string;
5050
CLOUDFLARE_SITE_TAG: string;
5151
PMTILES_SERVER_ZONE: string;
52+
R2_TILES_BUCKET: string;
5253
}
5354
}

0 commit comments

Comments
 (0)