huanld/pygeoapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update TinyDBCatalogue based on OARec updates

Browse Source
This commit is contained in:
Tom Kralidis
2023-03-28 02:08:32 -04:00
parent 4689daab19
commit 21ad9ea8a1
4 changed files with 43 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pygeoapi/provider/tinydb_.py
+18 -21
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2022 Tom Kralidis
# Copyright (c) 2023 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -32,11 +32,9 @@ import re # noqa
import os
import uuid
from shapely.geometry import shape
from tinydb import TinyDB, Query, where
from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError,
ProviderQueryError,
ProviderItemNotFoundError)
LOGGER = logging.getLogger(__name__)
@@ -91,7 +89,7 @@ class TinyDBCatalogueProvider(BaseProvider):
return fields
for p in r['properties'].keys():
if p not in self.excludes + ['extent']:
if p not in self.excludes:
fields[p] = {'type': 'string'}
fields['q'] = {'type': 'string'}
@@ -135,26 +133,30 @@ class TinyDBCatalogueProvider(BaseProvider):
if bbox:
LOGGER.debug('processing bbox parameter')
bbox_as_string = ','.join(str(s) for s in bbox)
QUERY.append(f"Q.properties.extent.spatial.bbox.test(bbox_intersects, '{bbox_as_string}')") # noqa
QUERY.append(f"Q.geometry.coordinates.test(bbox_intersects, '{bbox_as_string}')") # noqa
if datetime_ is not None:
LOGGER.debug('processing datetime parameter')
if self.time_field is None:
LOGGER.error('time_field not enabled for collection')
raise ProviderQueryError()
LOGGER.error('Using default time property')
time_field2 = 'time'
else:
LOGGER.error(f'Using properties.{self.time_field}')
time_field2 = f"properties['{self.time_field}']"
if '/' in datetime_: # envelope
LOGGER.debug('detected time range')
time_begin, time_end = datetime_.split('/')
if time_begin != '..':
QUERY.append(f"(Q.properties[self.time_field]>='{time_begin}')") # noqa
QUERY.append(f"(Q.{time_field2}>='{time_begin}')") # noqa
if time_end != '..':
QUERY.append(f"(Q.properties[self.time_field]<='{time_end}')") # noqa
QUERY.append(f"(Q.{time_field2}<='{time_end}')") # noqa
else: # time instant
LOGGER.debug('detected time instant')
QUERY.append(f"(Q.properties[self.time_field]=='{datetime_}')") # noqa
QUERY.append(f"(Q.{time_field2}=='{datetime_}')") # noqa
if properties:
LOGGER.debug('processing properties')
@@ -263,17 +265,6 @@ class TinyDBCatalogueProvider(BaseProvider):
LOGGER.debug('Missing title and description')
json_data['properties']['_metadata_anytext'] = ''
# Create properties.extent.spatial.bbox if not existing
if "geometry" in json_data and json_data["geometry"] is not None and \
("extent" not in json_data["properties"] or
"spatial" not in json_data["properties"]["extent"]):
if "extent" not in json_data["properties"]:
json_data["properties"]["extent"] = {}
if "spatial" not in json_data["properties"]["extent"]:
json_data["properties"]["extent"]["spatial"] = {}
json_data["properties"]["extent"]["spatial"]["bbox"] = \
[list(shape(json_data["geometry"]).bounds)]
LOGGER.debug(f'Inserting data with identifier {identifier}')
result = self.db.insert(json_data)
@@ -338,7 +329,13 @@ def bbox_intersects(record_bbox, input_bbox):
:returns: `bool` of whether the record_bbox intersects input_bbox
"""
bbox1 = record_bbox[0]
bbox1 = [
record_bbox[0][0][0],
record_bbox[0][0][1],
record_bbox[0][2][0],
record_bbox[0][2][1]
]
bbox2 = [float(c) for c in input_bbox.split(',')]
LOGGER.debug(f'Record bbox: {bbox1}')
tests/data/open.canada.ca/sample-records.tinydb
+1 -1
View File
File diff suppressed because one or more lines are too long
tests/load_tinydb_records.py
+19 -22
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2022 Tom Kralidis
# Copyright (c) 2023 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -63,26 +63,31 @@ def contact2party(ci: CI_ResponsibleParty) -> dict:
party = {
'contactInfo': {
'address': {
'main': {}
'office': {}
}
}
}
party['name'] = ci.name or ci.position
if ci.phone:
party['contactInfo']['phone'] = ci.phone
party['contactInfo']['phone'] = {
'office': ci.phone
}
if ci.email:
party['contactInfo']['email'] = ci.email
party['contactInfo']['email'] = {
'office': ci.email
}
if ci.address:
party['contactInfo']['address']['main']['deliveryPoint'] = ci.address
party['contactInfo']['address']['office']['deliveryPoint'] = ci.address
if ci.city:
party['contactInfo']['address']['main']['city'] = ci.city
party['contactInfo']['address']['office']['city'] = ci.city
if ci.region:
party['contactInfo']['address']['main']['administrativeArea'] = ci.region # noqa
party['contactInfo']['address']['office']['administrativeArea'] = ci.region # noqa
if ci.postcode:
party['contactInfo']['address']['main']['postalCode'] = ci.postcode
party['contactInfo']['address']['office']['postalCode'] = ci.postcode
if ci.country:
party['contactInfo']['address']['main']['country'] = ci.country
party['contactInfo']['address']['office']['country'] = ci.country
if ci.onlineresource:
party['contactInfo']['url'] = {
'href': ci.onlineresource.url,
@@ -124,7 +129,8 @@ def get_anytext(bag: Union[list, str]) -> str:
for xml_file in xml_dir.glob('*.xml'):
m = MD_Metadata(etree.parse(xml_file))
print(xml_file)
m = MD_Metadata(etree.parse(str(xml_file)))
_raw_metadata = m.xml.decode('utf-8')
_anytext = get_anytext(_raw_metadata)
@@ -191,6 +197,7 @@ for xml_file in xml_dir.glob('*.xml'):
'http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core'
],
'type': 'Feature',
'time': [te_begin, te_end],
'geometry': {
'type': 'Polygon',
'coordinates': [[
@@ -202,8 +209,8 @@ for xml_file in xml_dir.glob('*.xml'):
]]
},
'properties': {
'recordCreated': issued,
'recordUpdated': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
'created': issued,
'updated': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
'type': type_,
'title': title,
'description': description,
@@ -213,16 +220,6 @@ for xml_file in xml_dir.glob('*.xml'):
'value': identifier
}],
'themes': themes,
'extent': {
'spatial': {
'bbox': [bbox],
'crs': bbox_crs
},
'temporal': {
'interval': [te_begin, te_end],
'trs': 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' # noqa
}
},
'_metadata-anytext': _anytext
},
'links': links
tests/test_tinydb_catalogue_provider.py
+5 -11
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2021 Tom Kralidis
# Copyright (c) 2023 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -86,7 +86,7 @@ def config(tmp_path):
'type': 'feature',
'data': tmp_file,
'id_field': 'externalId',
'time_field': 'recordCreated'
'time_field': 'created'
}
@@ -95,7 +95,7 @@ def test_query(config):
fields = p.get_fields()
assert len(fields) == 9
assert fields['recordCreated']['type'] == 'string'
assert fields['created']['type'] == 'string'
assert fields['title']['type'] == 'string'
assert fields['q']['type'] == 'string'
@@ -189,14 +189,8 @@ def test_transactions_create_no_id(config, data_no_id):
assert new_id is not None
data_got = p.get(new_id)
assert data_got["id"] == new_id
expected_properties = json.loads(data_no_id)["properties"]
expected_properties["extent"] = {}
expected_properties["extent"]["spatial"] = {}
expected_properties["extent"]["spatial"]["bbox"] = \
[[100.0, 0.0, 101.0, 1.0]]
assert data_got["properties"] == expected_properties
assert data_got["geometry"] == json.loads(data_no_id)["geometry"]
assert data_got['id'] == new_id
assert data_got['geometry'] == json.loads(data_no_id)['geometry']
assert p.update(new_id, json.dumps(data_got))