Merge branch 'dev' into polishing_more

.gitignore

@@ -1,6 +1,3 @@
-access_log
-error_log*
-
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Caddyfile

@@ -1,3 +0,0 @@
-geo.speckle.systems {
-	reverse_proxy localhost:8000
-}

README.md

@@ -81,7 +81,7 @@ Then you can add it to the base map (e.g. using Leaflet and OpenStreetMap basema
         loadSpeckleData();
 
         async function loadSpeckleData() => {
-            var speckle_model_url = 'https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons';
+            var speckle_model_url = 'https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons';
             const speckle_data = await fetch(speckle_model_url, {
                 headers: {'Accept': 'application/geo+json'}
             }).then(response => response.json());

example-config.yml

@@ -128,4 +128,4 @@ resources:
     hello-world:
         type: process
         processor:
-            name: HelloWorld
+            name: HelloWorld

pygeoapi/api/__init__.py

@@ -75,7 +75,7 @@ LOGGER = logging.getLogger(__name__)
 #: Return headers for requests (e.g:X-Powered-By)
 HEADERS = {
     'Content-Type': 'application/json',
-    # 'X-Powered-By': f'pygeoapi {__version__}'
+    'X-Powered-By': f'pygeoapi {__version__}'
 }
 
 CHARSET = ['utf-8']

pygeoapi/flask_app.py

@@ -168,21 +168,6 @@ def execute_from_flask(api_function, request: Request, *args,
     return get_response((headers, status, content))
 
 
-def generate():
-    collection_id = "speckle"
-
-    yield loading_screen().data
-    
-    try:
-        browser_response = execute_from_flask(itemtypes_api.get_collection_items,
-                            request, collection_id,
-                            skip_valid_check=True)
-        yield browser_response.data
-
-    except Exception as ex:
-        yield error_screen(ex).data
-
-
 @BLUEPRINT.route('/')
 def landing_page():
     """
@@ -191,6 +176,8 @@ def landing_page():
     :returns: HTTP response
     """
     
+    collection_id = "speckle"
+
     agent = request.headers.get('User-Agent')
     # Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36
     # Mozilla/5.0 QGIS/32815/Windows 10 Version 2009
@@ -200,11 +187,7 @@ def landing_page():
     browser_list = ["Chrome", "Safari", "Firefox", "Edg/", "Trident/"]
     print(agent)
     
-    request.url += f"&userAgent={agent}"
-    CONFIG = get_config(request=request)
-    api_ = API(CONFIG, OPENAPI)
-    
-    if agent is not None and ("YaBrowser/" in agent or "yandex" in agent.lower()):
+    if "YaBrowser/" in agent:
         raise ValueError("Your browser is not supported.")
     for br in browser_list:
         if br in agent:
@@ -213,6 +196,18 @@ def landing_page():
     
     # if requested from the browser, return this, otherwise ignore IF statement
     if request.method == 'GET' and browser_agent:  # list items
+        
+        def generate():
+            yield loading_screen().data
+            try:
+                browser_response = execute_from_flask(itemtypes_api.get_collection_items,
+                                    request, collection_id,
+                                    skip_valid_check=True)
+                yield browser_response.data
+
+            except Exception as ex:
+                yield error_screen(ex).data
+
         return Response(stream_with_context(generate()))
     
     return get_response(api_.landing_page(request))
@@ -299,17 +294,10 @@ def collections(collection_id=None):
     :returns: HTTP response
     """
     
+    # raise NotImplementedError()
     return get_response(api_.describe_collections(request, collection_id))
 
 
-@BLUEPRINT.route('/speckle')
-def speckle_collection():
-    
-    collection_id="speckle"
-
-    return collection_items(collection_id=collection_id)
-
-
 @BLUEPRINT.route('/collections/<path:collection_id>/schema')
 def collection_schema(collection_id):
     """

pygeoapi/provider/speckle.py

@@ -119,16 +119,11 @@ class SpeckleProvider(BaseProvider):
         self.speckle_url = self.url.lower().split("speckleurl=")[-1].split("&")[0].split("@")[0].split("?")[0]
 
         self.speckle_data = None
-        self.project_name = ""
         self.model_name = ""
 
         self.crs = None
         self.crs_dict = None
 
-        self.commit_gis = False
-        self.url_params = {"url_data_type":"", "url_preserve_attributes":"", "url_crs_authid":"", "url_lat":"","url_lon":"","url_north_degrees":"","url_limit":""}
-        self.times = {}
-
         self.requested_data_type: str = "polygons (default)" # points, lines, polygons, projectcomments
         self.preserve_attributes: str = "true (default)"
         self.lat: float = 48.76755913928929 #51.52486388756923
@@ -136,7 +131,6 @@ class SpeckleProvider(BaseProvider):
         self.north_degrees: float = 0
         self.crs_authid = ""
         self.limit = 10000
-        self.user_agent = ""
 
         self.missing_url = ""
         self.limit_message = ""
@@ -344,8 +338,7 @@ class SpeckleProvider(BaseProvider):
     def load_speckle_data(self: str) -> Dict:
         """Receive and process Speckle data, return geojson."""
 
-        from datetime import datetime, timezone
-        from pygeoapi.provider.speckle_utils.server_utils import get_stream_branch, get_client, get_comments, set_actions
+        from pygeoapi.provider.speckle_utils.server_utils import get_stream_branch, get_client, get_comments
 
         from specklepy.objects.base import Base
         from specklepy.logging.exceptions import SpeckleException
@@ -375,7 +368,6 @@ class SpeckleProvider(BaseProvider):
             comments = {}
 
         # set the Model name
-        self.project_name = stream['name']
         self.model_name = branch['name']
 
         commit = branch["commits"]["items"][0]
@@ -386,7 +378,6 @@ class SpeckleProvider(BaseProvider):
             raise SpeckleException("Transport not found")
 
         # receive commit
-        set_actions(self, client)
         try:
             commit_obj = operations.receive(objId, transport, None)
         except Exception as ex:
@@ -400,9 +391,10 @@ class SpeckleProvider(BaseProvider):
             message="Received commit in pygeoapi",
         )
 
-        print(f"____{datetime.now().astimezone(timezone.utc)} _Rendering model '{branch['name']}' of the project '{stream['name']}'")
+        print(f"Rendering model '{branch['name']}' of the project '{stream['name']}'")
         speckle_data = self.traverse_data(commit_obj, comments)
 
+
         speckle_data["features"].extend(speckle_data["comments"])
         speckle_data["comments"] = []
 
@@ -481,10 +473,7 @@ class SpeckleProvider(BaseProvider):
             sorted_list[i]["properties"]["FID"] = i+1 
         data['features'] = sorted_list
         time2 = datetime.now()
-        
-        time_operation = (time2-time1).total_seconds()
-        self.times["time_sort"] = time_operation
-        # print(f"Sorting time: {time_operation}")
+        print(f"Sorting time: {(time2-time1).total_seconds()}")
 
         return data
     

pygeoapi/provider/speckle_utils/coords_utils.py

@@ -13,10 +13,7 @@ def reproject_bulk(self, all_coords: List[List[List[float]]], all_coord_counts:
     time1 = datetime.now()
     flat_coords = reproject_2d_coords_list(self, all_coords)
     time2 = datetime.now()
-
-    time_operation = (time2-time1).total_seconds()
-    self.times["time_reproject"] = time_operation
-    # print(f"Reproject time: {time_operation}")
+    print(f"Reproject time: {(time2-time1).total_seconds()}")
 
     # define type of features
     feat_coord_group_is_multi = [True if None in x else False for x in all_coord_counts]
@@ -64,10 +61,7 @@ def reproject_bulk(self, all_coords: List[List[List[float]]], all_coord_counts:
             geometry["coordinates"].extend(polygon_parts)
     
     time3 = datetime.now()
-
-    time_operation = (time3-time2).total_seconds()
-    self.times["time_reconstruct_geometry"] = time_operation
-    # print(f"Construct back geometry time: {time_operation}")
+    print(f"Construct back geometry time: {(time3-time2).total_seconds()}")
 
 def reproject_2d_coords_list(self, coords_in: List[List[float]]) -> List[List[float]]:
     """Return coordinates in a CRS of SpeckleProvider."""

pygeoapi/provider/speckle_utils/crs_utils.py

@@ -73,7 +73,6 @@ def get_set_crs_settings(self: "SpeckleProvider", commit_obj: "Base", context_li
             offset_y = crs["offset_y"]
             self.north_degrees = crs["rotation"]
             create_crs_from_wkt(self, crs["wkt"])
-            self.commit_gis = True
 
             if self.crs.to_authority() is not None:
                 data["model_crs"] = f"{self.crs.to_authority()}, {self.crs.name} "

pygeoapi/provider/speckle_utils/feature_utils.py

@@ -13,7 +13,7 @@ def initialize_features(self: "SpeckleProvider", all_coords, all_coord_counts, d
     from specklepy.objects.graph_traversal.traversal import TraversalContext
     from specklepy.objects.other import Collection
 
-    # print(f"Creating features..")
+    print(f"Creating features..")
     time1 = datetime.now()
     
     all_props = []
@@ -172,10 +172,7 @@ def initialize_features(self: "SpeckleProvider", all_coords, all_coord_counts, d
         raise ValueError("No supported features found")
     
     time2 = datetime.now()
-
-    time_operation = (time2-time1).total_seconds()
-    self.times["time_creating_features"] = time_operation
-    # print(f"Creating features time: {time_operation}")
+    print(f"Creating features time: {(time2-time1).total_seconds()}")
 
 def get_feature_bbox(coords) -> List[float]:
     """Get min max coordinates of the feature."""

pygeoapi/provider/speckle_utils/patch/server.py

@@ -168,23 +168,8 @@ class ServerTransport(AbstractTransport):
         # iter through returned objects saving them as we go
         target_transport.begin_write()
         all_lines = [line for _,line in enumerate(lines)]
-
-        # fix wrongly split lines 
         for i, line in enumerate(all_lines):
-            if line and i!= 0 and (len(line)<=10 or'"id": "' not in line[:10]):
-                # find the last line with ID
-                matching_index = -1
-                for k, id_line in enumerate(all_lines):
-                    if k<i and '"id": "' in id_line:
-                        matching_index = k
-                    if k==i:
-                        break
-                if matching_index != -1:
-                    all_lines[matching_index] += "},{" + line
-                    all_lines[i] = ""
-
-        for i, line in enumerate(all_lines):
-            if line and len(line)>10 and '"id": "' in line[:10]:
+            if line:
                 hash = line.split('"id": "')[1].split('"')[0]
                 obj = "{" + line + "}"
                 if i==0:

pygeoapi/provider/speckle_utils/server_utils.py

@@ -216,15 +216,4 @@ def get_attachment(project_id: str, attachment_id: str, attachment_name: str) ->
         raise Exception(
             f"Request not successful: Response code {r.status_code}"
         )
-
-def set_actions(self: "SpeckleProvider", client: "SpeckleClient"):
-    from specklepy.logging.metrics import track
-    try:
-        full_dict = {**self.url_params, **self.times}
-        full_dict["GIS commit"] = self.commit_gis
-        full_dict["model"] = f"{self.project_name}, {self.model_name}"
-        full_dict["time_TOTAL"] = sum([x[1] for x in self.times.items()])
-        full_dict["model_url"] = self.speckle_url
-        track("GEO receive", client.account, full_dict)
-    except:
-        pass
+    

pygeoapi/provider/speckle_utils/url_utils.py

@@ -31,7 +31,6 @@ def get_set_url_parameters(self: "SpeckleProvider"):
                     requested_data_type = item.split("datatype=")[1]
                     if requested_data_type in ["points", "lines", "polygons", "projectcomments"]:
                         self.requested_data_type = requested_data_type
-                        self.url_params["url_data_type"] = requested_data_type
                 except:
                     raise ValueError(f"Provide valid dataType parameter (points/lines/polygons/projectcomments): {item}")     
             
@@ -40,7 +39,6 @@ def get_set_url_parameters(self: "SpeckleProvider"):
                     preserve_attributes = item.split("preserveattributes=")[1]
                     if preserve_attributes in ["true", "false"]:
                         self.preserve_attributes = preserve_attributes
-                        self.url_params["url_preserve_attributes"] = preserve_attributes
                 except:
                     ValueError(f"Provide valid preserverAttributes parameter (true/false): {item}")
 
@@ -49,27 +47,23 @@ def get_set_url_parameters(self: "SpeckleProvider"):
                 if isinstance(crs_authid, str) and len(crs_authid)>3:
                     crsauthid = True
                     self.crs_authid = crs_authid
-                    self.url_params["url_crs_authid"] = crs_authid
 
             elif "lat=" in item:
                 try:
                     lat = float(item.split("lat=")[1])
                     self.lat = lat
-                    self.url_params["url_lat"] = lat
                 except:
                     raise ValueError(f"Invalid Lat input, must be numeric: {item}")
             elif "lon=" in item:
                 try:
                     lon = float(item.split("lon=")[1])
                     self.lon = lon
-                    self.url_params["url_lon"] = lon
                 except:
                     raise ValueError(f"Invalid Lon input, must be numeric: {item}")
             elif "northdegrees=" in item:
                 try:
                     north_degrees = float(item.split("northdegrees=")[1])
                     self.north_degrees = north_degrees
-                    self.url_params["url_north_degrees"] = north_degrees
                 except:
                     raise ValueError(f"Invalid northDegrees input, must be numeric: {item}")    
             elif "limit=" in item:
@@ -77,15 +71,6 @@ def get_set_url_parameters(self: "SpeckleProvider"):
                     limit = int(item.split("limit=")[1])
                     if limit>0: 
                         self.limit = limit
-                        self.url_params["url_limit"] = limit
-                except:
-                    ValueError(f"Invalid limit input, must be a positive integer: {item}")
-                 
-            elif "useragent=" in item:
-                try:
-                    agent = item.split("useragent=")[1]
-                    self.user_agent = agent
-                    self.url_params["user_agent"] = agent
                 except:
                     ValueError(f"Invalid limit input, must be a positive integer: {item}")
                 

pygeoapi/templates/_base.html

@@ -71,7 +71,7 @@
     <meta name="language" content="{{ config['server']['language'] }}">
     <meta name="description" content="{{ config['metadata']['identification']['title'] }}">
     <meta name="keywords" content="{{ config['metadata']['identification']['keywords']|join(',') }}">
-    <link rel="shortcut icon" href="https://github.com/specklesystems/pygeoapi/blob/dev/pygeoapi/static/img/speckle_geo.png" type="image/x-icon">
+    <link rel="shortcut icon" href="{{ config['server']['url'] }}/static/img/speckle_geo.png" type="image/x-icon">
     <link rel="stylesheet" href="https://unpkg.com/bootstrap@5.1.3/dist/css/bootstrap.min.css">
     <link rel="stylesheet" href="{{ config['server']['url'] }}/static/css/default.css">
     <!--[if lt IE 9]>
@@ -109,9 +109,11 @@
               title="{{ config['metadata']['identification']['title'] }}" style="height:30px;vertical-align: middle;" /> 
             <b style="text-align:left;padding-left: 10px;">Speckle</b> 
         </a>
-        <a href="https://geo.speckle.systems/" target="_blank" style="text-align:left;padding-left: 10px;" class="d-flex align-items-center mb-3 mb-md-0 me-md-auto text-dark text-decoration-none">
+        <p style="text-align:left;padding-left: 10px;" class="d-flex align-items-center mb-3 mb-md-0 me-md-auto text-dark text-decoration-none">
+          <a href="https://geo.speckle.systems/" target="_blank" style="text-align:left;padding-left: 10px;" class="d-flex align-items-center mb-3 mb-md-0 me-md-auto text-dark text-decoration-none">
             > Geolocating your data 
-        </a>
+            </a>
+        </p> 
         {% if (data["model"] and data["model"]!="") %}
         <a href="{{data['speckle_project_url']}}" target="_blank" style="text-align:left;padding-left: 10px;" class="d-flex align-items-center mb-3 mb-md-0 me-md-auto text-dark text-decoration-none">
           > {{data["project"]}} >

pygeoapi/templates/collections/items/index.html

@@ -335,10 +335,7 @@
 {% block extrafoot %}
 
     <script>
-      try {
-        document.getElementById("loading_screen").remove();
-      }
-      catch(err) {}
+      document.getElementById("loading_screen").remove();
 
       // attach even to modeSwitch btn
       document.getElementById("modeSwitch").onclick = switchMode;

pygeoapi/templates/error_screen.html

@@ -9,10 +9,7 @@
 </body>
 
 <script>
-  try {
-    document.getElementById("loading_screen").remove();
-  }
-  catch(err) {}
+  document.getElementById("loading_screen").remove();
 </script>
 
 </html>

pygeoapi/templates/exception.html

@@ -6,9 +6,6 @@
       <p>{{ data['description'] }}</p>
     </section>
     <script>
-      try {
-        document.getElementById("loading_screen").remove();
-      }
-      catch(err) {}
+      document.getElementById("loading_screen").remove();
     </script>
 {% endblock %}

requirements.txt

@@ -17,4 +17,3 @@ shapely
 SQLAlchemy<2.0.0
 tinydb
 unicodecsv
-specklepy==2.19.6

speckle_demos/demo_leaflet_all_data.html

@@ -86,7 +86,7 @@
             });
 
             // construnt data URL
-            var link = "https://geo.speckle.systems/speckle/?speckleUrl=";
+            var link = "https://geo.speckle.systems/?speckleUrl=";
             if (document.getElementById("speckle_model").value!=""){
                 link += document.getElementById("speckle_model").value.replace(" ", "")
             }

speckle_demos/demo_leaflet_comments.html

@@ -34,7 +34,7 @@
         ));
         
         (async () => {
-                const speckle_data = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
+                const speckle_data = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
                    headers: {
                      'Accept': 'application/geo+json'
                    }

speckle_demos/demo_leaflet_masterplan.html

@@ -34,7 +34,7 @@
         ));
         
         (async () => {
-                const speckle_data = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons', {
+                const speckle_data = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons', {
                    headers: {
                      'Accept': 'application/geo+json'
                    }

speckle_demos/demo_maptalks_comments.html

@@ -32,7 +32,7 @@
       
              
       (async () => {
-                const speckle_data2 = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/37d93c5d32&preserveAttributes=true', {
+                const speckle_data2 = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/37d93c5d32&preserveAttributes=true', {
                 //const speckle_data = await fetch('http://localhost:5000/?speckleUrl=https://app.speckle.systems/projects/64753f52b7/models/338b386787&limit=1000000&lat=-0.031405&lon=109.335828&preserveAttributes=true', {
                    headers: {
                      'Accept': 'application/geo+json'
@@ -155,7 +155,7 @@
                 
                 //map.fitBounds(speckle_layer.getBounds())
 
-                const speckle_data = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
+                const speckle_data = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
                 //const speckle_data = await fetch('http://localhost:5000/?speckleUrl=https://app.speckle.systems/projects/64753f52b7/models/338b386787&limit=1000000&lat=-0.031405&lon=109.335828&preserveAttributes=true', {
                    headers: {
                      'Accept': 'application/geo+json'

speckle_demos/demo_maptalks_polygons.html

@@ -30,12 +30,12 @@
       });
       
       (async () => {
-                //const speckle_data = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
-                //var speckle_url = 'http://localhost:5000/speckle/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/9c43d7569c&limit=1000000&datatype=polygons&preserveattributes=false';
+                //const speckle_data = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
+                //var speckle_url = 'http://localhost:5000/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/9c43d7569c&limit=1000000&datatype=polygons&preserveattributes=false';
                 // https://app.speckle.systems/projects/5feae56049/models/01c4183677
-                var speckle_url = 'https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/01c4183677&limit=1000000&datatype=polygons&preserveattributes=true';
+                var speckle_url = 'http://localhost:5000/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/01c4183677&limit=1000000&datatype=polygons&preserveattributes=true';
                 
-                //var speckle_url = 'https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/9c43d7569c&northDegrees=-30&preserveAttributes=true';
+                //var speckle_url = 'https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/9c43d7569c&northDegrees=-30&preserveAttributes=true';
                 const speckle_data = await fetch(speckle_url, {
                    headers: {
                      'Accept': 'application/geo+json'

speckle_demos/demo_openlayers_masterplan.html

@@ -65,7 +65,7 @@
 
         //////// add Speckle layer
         (async () => {
-            const geojson = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons', {
+            const geojson = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons', {
                 headers: {
                 'Accept': 'application/geo+json'
                 }

start.sh

@@ -1,8 +0,0 @@
-source .venv/bin/activate
-
-export PYGEOAPI_CONFIG="example-config.yml"
-export PYGEOAPI_OPENAPI="example-openapi.yml"
-export MAPTILER_KEY_SPECKLE="qam9vwl7bVk5tW1oZu46"
-export PORT=8000
-
-gunicorn pygeoapi.flask_app:APP --timeout 100000 --access-logfile access_log --error-logfile error_log --capture-output