request url

Exceptions and forwarding

.gitignore

@@ -1,3 +1,6 @@
+access_log
+error_log*
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Caddyfile

@@ -0,0 +1,3 @@
+geo.speckle.systems {
+	reverse_proxy localhost:8000
+}

README.md

@@ -81,7 +81,7 @@ Then you can add it to the base map (e.g. using Leaflet and OpenStreetMap basema
         loadSpeckleData();
 
         async function loadSpeckleData() => {
-            var speckle_model_url = 'https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons';
+            var speckle_model_url = 'https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons';
             const speckle_data = await fetch(speckle_model_url, {
                 headers: {'Accept': 'application/geo+json'}
             }).then(response => response.json());

example-config.yml

@@ -128,4 +128,4 @@ resources:
     hello-world:
         type: process
         processor:
-            name: HelloWorld
+            name: HelloWorld

pygeoapi/api/__init__.py

@@ -75,7 +75,7 @@ LOGGER = logging.getLogger(__name__)
 #: Return headers for requests (e.g:X-Powered-By)
 HEADERS = {
     'Content-Type': 'application/json',
-    'X-Powered-By': f'pygeoapi {__version__}'
+    # 'X-Powered-By': f'pygeoapi {__version__}'
 }
 
 CHARSET = ['utf-8']

pygeoapi/flask_app.py

@@ -168,6 +168,21 @@ def execute_from_flask(api_function, request: Request, *args,
     return get_response((headers, status, content))
 
 
+def generate():
+    collection_id = "speckle"
+
+    yield loading_screen().data
+    
+    try:
+        browser_response = execute_from_flask(itemtypes_api.get_collection_items,
+                            request, collection_id,
+                            skip_valid_check=True)
+        yield browser_response.data
+
+    except Exception as ex:
+        yield error_screen(ex).data
+
+
 @BLUEPRINT.route('/')
 def landing_page():
     """
@@ -176,8 +191,6 @@ def landing_page():
     :returns: HTTP response
     """
     
-    collection_id = "speckle"
-
     agent = request.headers.get('User-Agent')
     # Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36
     # Mozilla/5.0 QGIS/32815/Windows 10 Version 2009
@@ -187,7 +200,11 @@ def landing_page():
     browser_list = ["Chrome", "Safari", "Firefox", "Edg/", "Trident/"]
     print(agent)
     
-    if "YaBrowser/" in agent:
+    request.url += f"&userAgent={agent}"
+    CONFIG = get_config(request=request)
+    api_ = API(CONFIG, OPENAPI)
+    
+    if agent is not None and ("YaBrowser/" in agent or "yandex" in agent.lower()):
         raise ValueError("Your browser is not supported.")
     for br in browser_list:
         if br in agent:
@@ -196,18 +213,6 @@ def landing_page():
     
     # if requested from the browser, return this, otherwise ignore IF statement
     if request.method == 'GET' and browser_agent:  # list items
-        
-        def generate():
-            yield loading_screen().data
-            try:
-                browser_response = execute_from_flask(itemtypes_api.get_collection_items,
-                                    request, collection_id,
-                                    skip_valid_check=True)
-                yield browser_response.data
-
-            except Exception as ex:
-                yield error_screen(ex).data
-
         return Response(stream_with_context(generate()))
     
     return get_response(api_.landing_page(request))
@@ -294,10 +299,17 @@ def collections(collection_id=None):
     :returns: HTTP response
     """
     
-    # raise NotImplementedError()
     return get_response(api_.describe_collections(request, collection_id))
 
 
+@BLUEPRINT.route('/speckle')
+def speckle_collection():
+    
+    collection_id="speckle"
+
+    return collection_items(collection_id=collection_id)
+
+
 @BLUEPRINT.route('/collections/<path:collection_id>/schema')
 def collection_schema(collection_id):
     """

pygeoapi/provider/speckle.py

@@ -119,11 +119,16 @@ class SpeckleProvider(BaseProvider):
         self.speckle_url = self.url.lower().split("speckleurl=")[-1].split("&")[0].split("@")[0].split("?")[0]
 
         self.speckle_data = None
+        self.project_name = ""
         self.model_name = ""
 
         self.crs = None
         self.crs_dict = None
 
+        self.commit_gis = False
+        self.url_params = {"url_data_type":"", "url_preserve_attributes":"", "url_crs_authid":"", "url_lat":"","url_lon":"","url_north_degrees":"","url_limit":""}
+        self.times = {}
+
         self.requested_data_type: str = "polygons (default)" # points, lines, polygons, projectcomments
         self.preserve_attributes: str = "true (default)"
         self.lat: float = 48.76755913928929 #51.52486388756923
@@ -131,6 +136,7 @@ class SpeckleProvider(BaseProvider):
         self.north_degrees: float = 0
         self.crs_authid = ""
         self.limit = 10000
+        self.user_agent = ""
 
         self.missing_url = ""
         self.limit_message = ""
@@ -338,7 +344,8 @@ class SpeckleProvider(BaseProvider):
     def load_speckle_data(self: str) -> Dict:
         """Receive and process Speckle data, return geojson."""
 
-        from pygeoapi.provider.speckle_utils.server_utils import get_stream_branch, get_client, get_comments
+        from datetime import datetime, timezone
+        from pygeoapi.provider.speckle_utils.server_utils import get_stream_branch, get_client, get_comments, set_actions
 
         from specklepy.objects.base import Base
         from specklepy.logging.exceptions import SpeckleException
@@ -368,6 +375,7 @@ class SpeckleProvider(BaseProvider):
             comments = {}
 
         # set the Model name
+        self.project_name = stream['name']
         self.model_name = branch['name']
 
         commit = branch["commits"]["items"][0]
@@ -378,6 +386,7 @@ class SpeckleProvider(BaseProvider):
             raise SpeckleException("Transport not found")
 
         # receive commit
+        set_actions(self, client)
         try:
             commit_obj = operations.receive(objId, transport, None)
         except Exception as ex:
@@ -391,10 +400,9 @@ class SpeckleProvider(BaseProvider):
             message="Received commit in pygeoapi",
         )
 
-        print(f"Rendering model '{branch['name']}' of the project '{stream['name']}'")
+        print(f"____{datetime.now().astimezone(timezone.utc)} _Rendering model '{branch['name']}' of the project '{stream['name']}'")
         speckle_data = self.traverse_data(commit_obj, comments)
 
-
         speckle_data["features"].extend(speckle_data["comments"])
         speckle_data["comments"] = []
 
@@ -473,7 +481,10 @@ class SpeckleProvider(BaseProvider):
             sorted_list[i]["properties"]["FID"] = i+1 
         data['features'] = sorted_list
         time2 = datetime.now()
-        print(f"Sorting time: {(time2-time1).total_seconds()}")
+        
+        time_operation = (time2-time1).total_seconds()
+        self.times["time_sort"] = time_operation
+        # print(f"Sorting time: {time_operation}")
 
         return data
     

pygeoapi/provider/speckle_utils/coords_utils.py

@@ -13,7 +13,10 @@ def reproject_bulk(self, all_coords: List[List[List[float]]], all_coord_counts:
     time1 = datetime.now()
     flat_coords = reproject_2d_coords_list(self, all_coords)
     time2 = datetime.now()
-    print(f"Reproject time: {(time2-time1).total_seconds()}")
+
+    time_operation = (time2-time1).total_seconds()
+    self.times["time_reproject"] = time_operation
+    # print(f"Reproject time: {time_operation}")
 
     # define type of features
     feat_coord_group_is_multi = [True if None in x else False for x in all_coord_counts]
@@ -61,7 +64,10 @@ def reproject_bulk(self, all_coords: List[List[List[float]]], all_coord_counts:
             geometry["coordinates"].extend(polygon_parts)
     
     time3 = datetime.now()
-    print(f"Construct back geometry time: {(time3-time2).total_seconds()}")
+
+    time_operation = (time3-time2).total_seconds()
+    self.times["time_reconstruct_geometry"] = time_operation
+    # print(f"Construct back geometry time: {time_operation}")
 
 def reproject_2d_coords_list(self, coords_in: List[List[float]]) -> List[List[float]]:
     """Return coordinates in a CRS of SpeckleProvider."""

pygeoapi/provider/speckle_utils/crs_utils.py

@@ -73,6 +73,7 @@ def get_set_crs_settings(self: "SpeckleProvider", commit_obj: "Base", context_li
             offset_y = crs["offset_y"]
             self.north_degrees = crs["rotation"]
             create_crs_from_wkt(self, crs["wkt"])
+            self.commit_gis = True
 
             if self.crs.to_authority() is not None:
                 data["model_crs"] = f"{self.crs.to_authority()}, {self.crs.name} "

pygeoapi/provider/speckle_utils/display_utils.py

@@ -165,11 +165,11 @@ def get_single_display_object(displayValForColor: List) -> "Base":
                 except IndexError:
                     break
         elif item is not None:
-            displayValForColor = item
+            return item
 
     mesh = Mesh.create(faces= faces, vertices=verts, colors=colors)
 
-    if len(displayValForColor)>0:
+    if isinstance(displayValForColor, List) and len(displayValForColor)>0:
         for prop in displayValForColor[0].get_member_names():
             if prop not in ["colors", "vertices", "faces"]:
                 mesh[prop] = getattr(displayValForColor[0], prop)

pygeoapi/provider/speckle_utils/feature_utils.py

@@ -13,7 +13,7 @@ def initialize_features(self: "SpeckleProvider", all_coords, all_coord_counts, d
     from specklepy.objects.graph_traversal.traversal import TraversalContext
     from specklepy.objects.other import Collection
 
-    print(f"Creating features..")
+    # print(f"Creating features..")
     time1 = datetime.now()
     
     all_props = []
@@ -172,7 +172,10 @@ def initialize_features(self: "SpeckleProvider", all_coords, all_coord_counts, d
         raise ValueError("No supported features found")
     
     time2 = datetime.now()
-    print(f"Creating features time: {(time2-time1).total_seconds()}")
+
+    time_operation = (time2-time1).total_seconds()
+    self.times["time_creating_features"] = time_operation
+    # print(f"Creating features time: {time_operation}")
 
 def get_feature_bbox(coords) -> List[float]:
     """Get min max coordinates of the feature."""

pygeoapi/provider/speckle_utils/patch/server.py

@@ -168,8 +168,23 @@ class ServerTransport(AbstractTransport):
         # iter through returned objects saving them as we go
         target_transport.begin_write()
         all_lines = [line for _,line in enumerate(lines)]
+
+        # fix wrongly split lines 
         for i, line in enumerate(all_lines):
-            if line:
+            if line and i!= 0 and (len(line)<=10 or'"id": "' not in line[:10]):
+                # find the last line with ID
+                matching_index = -1
+                for k, id_line in enumerate(all_lines):
+                    if k<i and '"id": "' in id_line:
+                        matching_index = k
+                    if k==i:
+                        break
+                if matching_index != -1:
+                    all_lines[matching_index] += "},{" + line
+                    all_lines[i] = ""
+
+        for i, line in enumerate(all_lines):
+            if line and len(line)>10 and '"id": "' in line[:10]:
                 hash = line.split('"id": "')[1].split('"')[0]
                 obj = "{" + line + "}"
                 if i==0:

pygeoapi/provider/speckle_utils/server_utils.py

@@ -216,4 +216,15 @@ def get_attachment(project_id: str, attachment_id: str, attachment_name: str) ->
         raise Exception(
             f"Request not successful: Response code {r.status_code}"
         )
-    
+
+def set_actions(self: "SpeckleProvider", client: "SpeckleClient"):
+    from specklepy.logging.metrics import track
+    try:
+        full_dict = {**self.url_params, **self.times}
+        full_dict["GIS commit"] = self.commit_gis
+        full_dict["model"] = f"{self.project_name}, {self.model_name}"
+        full_dict["time_TOTAL"] = sum([x[1] for x in self.times.items()])
+        full_dict["model_url"] = self.speckle_url
+        track("GEO receive", client.account, full_dict)
+    except:
+        pass

pygeoapi/provider/speckle_utils/url_utils.py

@@ -31,6 +31,7 @@ def get_set_url_parameters(self: "SpeckleProvider"):
                     requested_data_type = item.split("datatype=")[1]
                     if requested_data_type in ["points", "lines", "polygons", "projectcomments"]:
                         self.requested_data_type = requested_data_type
+                        self.url_params["url_data_type"] = requested_data_type
                 except:
                     raise ValueError(f"Provide valid dataType parameter (points/lines/polygons/projectcomments): {item}")     
             
@@ -39,6 +40,7 @@ def get_set_url_parameters(self: "SpeckleProvider"):
                     preserve_attributes = item.split("preserveattributes=")[1]
                     if preserve_attributes in ["true", "false"]:
                         self.preserve_attributes = preserve_attributes
+                        self.url_params["url_preserve_attributes"] = preserve_attributes
                 except:
                     ValueError(f"Provide valid preserverAttributes parameter (true/false): {item}")
 
@@ -47,23 +49,27 @@ def get_set_url_parameters(self: "SpeckleProvider"):
                 if isinstance(crs_authid, str) and len(crs_authid)>3:
                     crsauthid = True
                     self.crs_authid = crs_authid
+                    self.url_params["url_crs_authid"] = crs_authid
 
             elif "lat=" in item:
                 try:
                     lat = float(item.split("lat=")[1])
                     self.lat = lat
+                    self.url_params["url_lat"] = lat
                 except:
                     raise ValueError(f"Invalid Lat input, must be numeric: {item}")
             elif "lon=" in item:
                 try:
                     lon = float(item.split("lon=")[1])
                     self.lon = lon
+                    self.url_params["url_lon"] = lon
                 except:
                     raise ValueError(f"Invalid Lon input, must be numeric: {item}")
             elif "northdegrees=" in item:
                 try:
                     north_degrees = float(item.split("northdegrees=")[1])
                     self.north_degrees = north_degrees
+                    self.url_params["url_north_degrees"] = north_degrees
                 except:
                     raise ValueError(f"Invalid northDegrees input, must be numeric: {item}")    
             elif "limit=" in item:
@@ -71,6 +77,15 @@ def get_set_url_parameters(self: "SpeckleProvider"):
                     limit = int(item.split("limit=")[1])
                     if limit>0: 
                         self.limit = limit
+                        self.url_params["url_limit"] = limit
+                except:
+                    ValueError(f"Invalid limit input, must be a positive integer: {item}")
+                 
+            elif "useragent=" in item:
+                try:
+                    agent = item.split("useragent=")[1]
+                    self.user_agent = agent
+                    self.url_params["user_agent"] = agent
                 except:
                     ValueError(f"Invalid limit input, must be a positive integer: {item}")
                 

pygeoapi/templates/_base.html

@@ -71,7 +71,7 @@
     <meta name="language" content="{{ config['server']['language'] }}">
     <meta name="description" content="{{ config['metadata']['identification']['title'] }}">
     <meta name="keywords" content="{{ config['metadata']['identification']['keywords']|join(',') }}">
-    <link rel="shortcut icon" href="{{ config['server']['url'] }}/static/img/speckle_geo.png" type="image/x-icon">
+    <link rel="shortcut icon" href="https://github.com/specklesystems/pygeoapi/blob/dev/pygeoapi/static/img/speckle_geo.png" type="image/x-icon">
     <link rel="stylesheet" href="https://unpkg.com/bootstrap@5.1.3/dist/css/bootstrap.min.css">
     <link rel="stylesheet" href="{{ config['server']['url'] }}/static/css/default.css">
     <!--[if lt IE 9]>
@@ -109,11 +109,9 @@
               title="{{ config['metadata']['identification']['title'] }}" style="height:30px;vertical-align: middle;" /> 
             <b style="text-align:left;padding-left: 10px;">Speckle</b> 
         </a>
-        <p style="text-align:left;padding-left: 10px;" class="d-flex align-items-center mb-3 mb-md-0 me-md-auto text-dark text-decoration-none">
-          <a href="https://geo.speckle.systems/" target="_blank" style="text-align:left;padding-left: 10px;" class="d-flex align-items-center mb-3 mb-md-0 me-md-auto text-dark text-decoration-none">
+        <a href="https://geo.speckle.systems/" target="_blank" style="text-align:left;padding-left: 10px;" class="d-flex align-items-center mb-3 mb-md-0 me-md-auto text-dark text-decoration-none">
             > Geolocating your data 
-            </a>
-        </p> 
+        </a>
         {% if (data["model"] and data["model"]!="") %}
         <a href="{{data['speckle_project_url']}}" target="_blank" style="text-align:left;padding-left: 10px;" class="d-flex align-items-center mb-3 mb-md-0 me-md-auto text-dark text-decoration-none">
           > {{data["project"]}} >

pygeoapi/templates/collections/items/index.html

@@ -335,7 +335,10 @@
 {% block extrafoot %}
 
     <script>
-      document.getElementById("loading_screen").remove();
+      try {
+        document.getElementById("loading_screen").remove();
+      }
+      catch(err) {}
 
       // attach even to modeSwitch btn
       document.getElementById("modeSwitch").onclick = switchMode;

pygeoapi/templates/error_screen.html

@@ -9,7 +9,10 @@
 </body>
 
 <script>
-  document.getElementById("loading_screen").remove();
+  try {
+    document.getElementById("loading_screen").remove();
+  }
+  catch(err) {}
 </script>
 
 </html>

pygeoapi/templates/exception.html

@@ -6,6 +6,9 @@
       <p>{{ data['description'] }}</p>
     </section>
     <script>
-      document.getElementById("loading_screen").remove();
+      try {
+        document.getElementById("loading_screen").remove();
+      }
+      catch(err) {}
     </script>
 {% endblock %}

requirements.txt

@@ -17,3 +17,4 @@ shapely
 SQLAlchemy<2.0.0
 tinydb
 unicodecsv
+specklepy==2.19.6

speckle_demos/demo_leaflet_all_data.html

@@ -86,7 +86,7 @@
             });
 
             // construnt data URL
-            var link = "https://geo.speckle.systems/?speckleUrl=";
+            var link = "https://geo.speckle.systems/speckle/?speckleUrl=";
             if (document.getElementById("speckle_model").value!=""){
                 link += document.getElementById("speckle_model").value.replace(" ", "")
             }

speckle_demos/demo_leaflet_comments.html

@@ -34,7 +34,7 @@
         ));
         
         (async () => {
-                const speckle_data = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
+                const speckle_data = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
                    headers: {
                      'Accept': 'application/geo+json'
                    }

speckle_demos/demo_leaflet_masterplan.html

@@ -34,7 +34,7 @@
         ));
         
         (async () => {
-                const speckle_data = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons', {
+                const speckle_data = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons', {
                    headers: {
                      'Accept': 'application/geo+json'
                    }

speckle_demos/demo_maptalks_comments.html

@@ -32,7 +32,7 @@
       
              
       (async () => {
-                const speckle_data2 = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/37d93c5d32&preserveAttributes=true', {
+                const speckle_data2 = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/37d93c5d32&preserveAttributes=true', {
                 //const speckle_data = await fetch('http://localhost:5000/?speckleUrl=https://app.speckle.systems/projects/64753f52b7/models/338b386787&limit=1000000&lat=-0.031405&lon=109.335828&preserveAttributes=true', {
                    headers: {
                      'Accept': 'application/geo+json'
@@ -155,7 +155,7 @@
                 
                 //map.fitBounds(speckle_layer.getBounds())
 
-                const speckle_data = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
+                const speckle_data = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
                 //const speckle_data = await fetch('http://localhost:5000/?speckleUrl=https://app.speckle.systems/projects/64753f52b7/models/338b386787&limit=1000000&lat=-0.031405&lon=109.335828&preserveAttributes=true', {
                    headers: {
                      'Accept': 'application/geo+json'

speckle_demos/demo_maptalks_polygons.html

@@ -30,12 +30,12 @@
       });
       
       (async () => {
-                //const speckle_data = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
-                //var speckle_url = 'http://localhost:5000/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/9c43d7569c&limit=1000000&datatype=polygons&preserveattributes=false';
+                //const speckle_data = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=projectcomments', {
+                //var speckle_url = 'http://localhost:5000/speckle/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/9c43d7569c&limit=1000000&datatype=polygons&preserveattributes=false';
                 // https://app.speckle.systems/projects/5feae56049/models/01c4183677
-                var speckle_url = 'http://localhost:5000/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/01c4183677&limit=1000000&datatype=polygons&preserveattributes=true';
+                var speckle_url = 'https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/01c4183677&limit=1000000&datatype=polygons&preserveattributes=true';
                 
-                //var speckle_url = 'https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/9c43d7569c&northDegrees=-30&preserveAttributes=true';
+                //var speckle_url = 'https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/5feae56049/models/9c43d7569c&northDegrees=-30&preserveAttributes=true';
                 const speckle_data = await fetch(speckle_url, {
                    headers: {
                      'Accept': 'application/geo+json'

speckle_demos/demo_openlayers_masterplan.html

@@ -65,7 +65,7 @@
 
         //////// add Speckle layer
         (async () => {
-            const geojson = await fetch('https://geo.speckle.systems/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons', {
+            const geojson = await fetch('https://geo.speckle.systems/speckle/?speckleUrl=https://app.speckle.systems/projects/344f803f81/models/5582ab673e&datatype=polygons', {
                 headers: {
                 'Accept': 'application/geo+json'
                 }

start.sh

@@ -0,0 +1,8 @@
+source .venv/bin/activate
+
+export PYGEOAPI_CONFIG="example-config.yml"
+export PYGEOAPI_OPENAPI="example-openapi.yml"
+export MAPTILER_KEY_SPECKLE="qam9vwl7bVk5tW1oZu46"
+export PORT=8000
+
+gunicorn pygeoapi.flask_app:APP --timeout 100000 --access-logfile access_log --error-logfile error_log --capture-output