Merge pull request #202 from specklesystems/dogukan/cnx-2515-fallback-to-json-for-scheduled-refresh

fallback to json download
clearer error message on visual
2025-09-11 15:02:39 +02:00 · 2025-09-11 15:00:00 +02:00 · 2025-09-11 07:13:22 +03:00 · 2025-09-11 06:57:59 +03:00 · 2025-09-10 13:22:41 +02:00 · 2025-09-09 19:19:21 +03:00
10 changed files with 176 additions and 158 deletions
@@ -247,7 +247,7 @@ GetByUrl.Icons = [
 Speckle = [
    // This is used when running the connector on an on-premises data gateway
    TestConnection = (path) => {"Speckle.GetUser", path},
-    // Authentication strategy
+    // Authentication strategy - OAuth only
    Authentication = [
        OAuth = [
            Label = "Speckle Account",
@@ -357,13 +357,6 @@ Speckle = [
                        ]
                in
                    result
-        ],
-        Key = [
-            KeyLabel = "Personal Access Token",
-            Label = "Private Project"
-        ],
-        Implicit = [
-            Label = "Public Project"
        ]
    ],
    Label = "Speckle"
@@ -1,7 +1,7 @@
 (server as text, optional query as text, optional variables as record) as record =>
    let
        // Enhanced credential retrieval with OAuth2 support
-        apiKey = try Extension.CurrentCredential()[Key] otherwise try Extension.CurrentCredential()[access_token] otherwise null,
+        apiKey = try Extension.CurrentCredential()[access_token] otherwise null,
        
        defaultQuery = "query {
                activeUser {
@@ -29,7 +29,7 @@
        versionId = parsedUrl[versionId],
        
        // get API key if available
-        apiKey = try Extension.CurrentCredential()[Key] otherwise null,
+        apiKey = try Extension.CurrentCredential()[access_token] otherwise null,
        
        // graphql query to get model info including root object id
        // includes specific version if provided
@@ -26,7 +26,7 @@ in
            parsedUrl = Parser(url),
            server = parsedUrl[baseUrl],
            
-            apiKey = try Extension.CurrentCredential()[Key] otherwise try Extension.CurrentCredential()[access_token] otherwise "",
+            apiKey = try Extension.CurrentCredential()[access_token] otherwise "",
            
            query = "query {
                    activeUser {
@@ -1,5 +1,5 @@
 (url as text) as list =>
-    try let
+    let
        // Import required functions
        GetModel = Extension.LoadFunction("GetModel.pqm"),
        Parser = Extension.LoadFunction("Parser.pqm"),
@@ -24,79 +24,145 @@
                            Detail = [File = fileName, Error = e]
                        ],

+        // Get required information
        modelInfo = GetModel(url),
        parsedUrl = Parser(url),
        userInfo = GetUser(url),
-
        apiKey = userInfo[Token],
-        
        userEmail = userInfo[UserEmail],
-
-        // get version from Speckle.pq - look GetVersion.pqm
        connectorVersion = GetVersion(),
-
        workspaceInfo = GetWorkspace(url),

-        // exchange powerful token for weak token via ds 
-        tokenExchangeData = Json.FromValue([
-            PowerfulToken = apiKey,
-            Scopes = {"profile:read", "streams:read", "users:read"},
-            ProjectId = parsedUrl[projectId]
-        ]),
-        
-        tokenExchangeResponse = Web.Contents(
-            "http://127.0.0.1:29364/auth/exchange-token",
-            [
-                Headers = [
-                    #"Content-Type" = "application/json",
-                    #"Method" = "POST"
-                ],
-                Content = tokenExchangeData,
-                ManualStatusHandling = {400, 401, 403, 404, 500}
-            ]
-        ),
-        
-        tokenExchangeJson = Json.Document(tokenExchangeResponse),
-        weakToken = tokenExchangeJson[token],
+        // Function to check if Desktop Service is available
+        IsDesktopServiceAvailable = () =>
+            try
+                let
+                    PingResponse = Web.Contents(
+                        "http://127.0.0.1:29364/ping",
+                        [
+                            Headers = [#"Method" = "GET"],
+                            ManualStatusHandling = {400, 401, 403, 404, 500, 502, 503, 504},
+                            Timeout = #duration(0, 0, 0, 2) // 2 second timeout for ping
+                        ]
+                    ),
+                    StatusCode = Value.Metadata(PingResponse)[Response.Status]
+                in
+                    StatusCode = 200
+            otherwise
+                false,

-        // prepare request data with weak token
-        requestData = Json.FromValue([
-            Url = url,
-            Server = parsedUrl[baseUrl],
-            Email = userEmail,
-            ProjectId = parsedUrl[projectId],
-            RootObjectId = modelInfo[rootObjectId],
-            SourceApplication = modelInfo[sourceApplication],
-            Token = weakToken,
-            Version = connectorVersion,
-            VersionId = parsedUrl[versionId],
-            WorkspaceId = workspaceInfo[workspaceId],
-            WorkspaceName = workspaceInfo[workspaceName],
-            WorkspaceLogo = workspaceInfo[workspaceLogo],
-            CanHideBranding = workspaceInfo[canHideBranding]
-        ]),
+        // Function to use Desktop Service approach (only called if available)
+        UseDesktopService = () =>
+            let
+                // exchange powerful token for weak token via ds 
+                tokenExchangeData = Json.FromValue([
+                    PowerfulToken = apiKey,
+                    Scopes = {"profile:read", "streams:read", "users:read"},
+                    ProjectId = parsedUrl[projectId],
+                    ServerUrl = parsedUrl[baseUrl]
+                ]),
+                
+                tokenExchangeResponse = Web.Contents(
+                    "http://127.0.0.1:29364/auth/exchange-token",
+                    [
+                        Headers = [
+                            #"Content-Type" = "application/json",
+                            #"Method" = "POST"
+                        ],
+                        Content = tokenExchangeData,
+                        ManualStatusHandling = {400, 401, 403, 404, 500}
+                    ]
+                ),
+                
+                tokenExchangeJson = Json.Document(tokenExchangeResponse),
+                weakToken = tokenExchangeJson[token],
+
+                // prepare request data with weak token
+                requestData = Json.FromValue([
+                    Url = url,
+                    Server = parsedUrl[baseUrl],
+                    Email = userEmail,
+                    ProjectId = parsedUrl[projectId],
+                    RootObjectId = modelInfo[rootObjectId],
+                    SourceApplication = modelInfo[sourceApplication],
+                    Token = weakToken,
+                    Version = connectorVersion,
+                    VersionId = parsedUrl[versionId],
+                    WorkspaceId = workspaceInfo[workspaceId],
+                    WorkspaceName = workspaceInfo[workspaceName],
+                    WorkspaceLogo = workspaceInfo[workspaceLogo],
+                    CanHideBranding = workspaceInfo[canHideBranding]
+                ]),
+                
+                // Send request to local server
+                Response = Web.Contents(
+                    "http://127.0.0.1:29364/download",
+                    [
+                        Headers = [
+                            #"Content-Type" = "application/json",
+                            #"Method" = "POST"
+                        ],
+                        Content = requestData,
+                        ManualStatusHandling = {400, 401, 403, 404, 500}
+                    ]
+                ),
+                
+                // Parse response
+                JsonResponse = Json.Document(Response)
+            in
+                JsonResponse,
+
+        // Function to fallback to direct JSON download from Speckle server
+        FallbackToDirectDownload = () =>
+            let
+                // Construct the direct object URL: {baseUrl}/objects/{projectId}/{rootObjectId}
+                objectUrl = Text.Combine({
+                    parsedUrl[baseUrl], 
+                    "/objects/", 
+                    parsedUrl[projectId], 
+                    "/", 
+                    modelInfo[rootObjectId]
+                }),
+                
+                // Download JSON directly from Speckle server
+                Response = Web.Contents(
+                    objectUrl,
+                    [
+                        Headers = [
+                            #"Authorization" = "Bearer " & apiKey,
+                            #"Accept" = "application/json"
+                        ],
+                        ManualStatusHandling = {400, 401, 403, 404, 500, 502, 503, 504}
+                    ]
+                ),
+                
+                // Check response status
+                StatusCode = Value.Metadata(Response)[Response.Status],
+                
+                // Parse JSON response if successful
+                JsonResponse = if StatusCode >= 200 and StatusCode < 300 then
+                    Json.Document(Response)
+                else
+                    error [
+                        Reason = "DirectDownloadFailed",
+                        Message = "Failed to download model data directly from Speckle server",
+                        Detail = [
+                            StatusCode = StatusCode, 
+                            ObjectUrl = objectUrl,
+                            ProjectId = parsedUrl[projectId],
+                            RootObjectId = modelInfo[rootObjectId]
+                        ]
+                    ]
+            in
+                JsonResponse,
+
+        // Check Desktop Service availability and choose approach
+        DesktopServiceAvailable = IsDesktopServiceAvailable(),
        
-        // Send request to local server
-        Response = Web.Contents(
-            "http://127.0.0.1:29364/download",
-            [
-                Headers = [
-                    #"Content-Type" = "application/json",
-                    #"Method" = "POST"
-                ],
-                Content = requestData,
-                ManualStatusHandling = {400, 401, 403, 404, 500}
-            ]
-        ),
-        
-        // Parse response
-        JsonResponse = Json.Document(Response)
+        FinalResult = if DesktopServiceAvailable then
+            UseDesktopService()
+        else
+            FallbackToDirectDownload()

    in
-        JsonResponse
-    otherwise
-        error [
-            Reason = "Desktop Service Not Available", 
-            Message = "Cannot connect to Speckle Desktop Service. Please ensure the Desktop Service is running and try again.",
-            Detail = "The Speckle Desktop Service must be running to load data from Speckle. Please start the Desktop Service application and refresh your data connection."
-        ]
+        FinalResult
@@ -1,5 +1,5 @@
 <template>
-  <div class="border">
+  <div>
    <transition name="slide-fade">
      <nav
        v-show="!visualStore.isNavbarHidden"
@@ -1,5 +1,5 @@
 import { formattingSettings as fs } from 'powerbi-visuals-utils-formattingmodel'
-import { ColorSelectorSettings, ColorSettings } from 'src/settings/colorSettings'
+import { ColorSettings } from 'src/settings/colorSettings'
 import { CameraSettings } from 'src/settings/cameraSettings'
 import { LightingSettings } from 'src/settings/lightingSettings'
 import { DataLoadingSettings } from 'src/settings/dataLoadingSettings'
@@ -8,8 +8,6 @@ export class SpeckleVisualSettingsModel extends fs.Model {
  // Building my visual formatting settings card
  public color: ColorSettings = new ColorSettings()

-  public colorSelector: ColorSelectorSettings = new ColorSelectorSettings()
-
  public dataLoading: DataLoadingSettings = new DataLoadingSettings()

  // public camera: CameraSettings = new CameraSettings()
@@ -26,7 +26,7 @@ export const useVisualStore = defineStore('visualStore', () => {
  const host = shallowRef<powerbi.extensibility.visual.IVisualHost>()
  const formattingSettings = ref<SpeckleVisualSettingsModel>()
  const loadingProgress = ref<LoadingProgress>(undefined)
-  const objectsFromStore = ref<object[]>(undefined)
+  const objectsFromStore = ref<object[][]>(undefined)

  // State tracking for toggle reset prevention
  const previousToggleState = ref<boolean | undefined>(undefined)
@@ -88,8 +88,14 @@ export const useVisualStore = defineStore('visualStore', () => {

  const setReceiveInfo = (newReceiveInfo: ReceiveInfo) => {
    receiveInfo.value = newReceiveInfo
-    // Only save receiveInfo to file in offline mode for persistence (contains token and metadata)
-    if (formattingSettings.value?.dataLoading.internalizeData.value) {
+    
+    // Always save receiveInfo to file for credentials persistence (contains token and metadata)
+    // This ensures weak tokens are available even when desktop service is unavailable
+    if (formattingSettings.value?.dataLoading.internalizeData.value && objectsFromStore.value) {
+      // If internalize is ON and we have objects, save both objects and receiveInfo together
+      writeObjectsToFile(objectsFromStore.value)
+    } else {
+      // Otherwise just save receiveInfo alone (credentials only)
      writeReceiveInfoToFile()
    }
  }
@@ -122,7 +128,7 @@ export const useVisualStore = defineStore('visualStore', () => {
    }
  }

-  const setObjectsFromStore = (newObjectsFromStore: object[]) => {
+  const setObjectsFromStore = (newObjectsFromStore: object[][]) => {
    objectsFromStore.value = newObjectsFromStore
  }

@@ -214,7 +220,7 @@ export const useVisualStore = defineStore('visualStore', () => {
          objectName: 'storedData',
          properties: {
            speckleObjects: compressedChunks,
-            receiveInfo: JSON.stringify(receiveInfo.value)
+            receiveInfo: JSON.stringify(receiveInfo.value) // Keep receiveInfo in sync when storing objects
          },
          selector: null
        }
@@ -170,13 +170,14 @@ async function getReceiveInfo(id) {
    const response = await fetch(`http://localhost:29364/user-info/${ids[0]}`)
    if (!response.body) {
      console.error('No response body')
-      return
+      return { desktopServiceError: true }
    }

    return await response.json()
  } catch (error) {
    console.log(error)
    console.log("User info couldn't retrieved from local server.")
+    return { desktopServiceError: true }
  }
 }

@@ -231,7 +232,11 @@ export async function processMatrixView(

  try {
    if (hasColorFilter) {
-      if (!localMatrixView[0].children || localMatrixView[0].children.length === 0 || !localMatrixView[0].children[0].values) {
+      if (
+        !localMatrixView[0].children ||
+        localMatrixView[0].children.length === 0 ||
+        !localMatrixView[0].children[0].values
+      ) {
        throw new Error('Matrix view structure is incomplete for color filter mode')
      }
      id = localMatrixView[0].children[0].values[0].value as unknown as string
@@ -258,7 +263,9 @@ export async function processMatrixView(
        // CRITICAL: Validate that internalized data matches current matrix data
        const internalizedRootId = (internalizedModelObjects[0][0] as any).id
        if (internalizedRootId !== id) {
-          console.log(`📁 Internalized data mismatch: stored=${internalizedRootId}, current=${id}. Using fresh data.`)
+          console.log(
+            `📁 Internalized data mismatch: stored=${internalizedRootId}, current=${id}. Using fresh data.`
+          )
          internalizedModelObjects = undefined // Clear internalized data - use fresh data instead
        } else {
          console.log(
@@ -270,7 +277,6 @@ export async function processMatrixView(
      }

      if (internalizedModelObjects && internalizedModelObjects.length > 0) {
-
        // Set dummy receiveInfo to prevent UI errors
        if (!visualStore.receiveInfo) {
          visualStore.setReceiveInfo({
@@ -288,7 +294,8 @@ export async function processMatrixView(
        }

        // Only reload if switching models or not already loaded
-        const needsReload = !visualStore.isViewerObjectsLoaded || visualStore.lastLoadedRootObjectId !== id
+        const needsReload =
+          !visualStore.isViewerObjectsLoaded || visualStore.lastLoadedRootObjectId !== id
        if (needsReload) {
          console.log('🔄 Forcing viewer reload for internalized data (model switch or first load)')
          visualStore.setViewerReloadNeeded()
@@ -321,7 +328,9 @@ export async function processMatrixView(

    // Get receive info from desktop service to populate visual store
    const receiveInfo = await getReceiveInfo(id)
-    if (receiveInfo) {
+    let desktopServiceUnavailable = false
+
+    if (receiveInfo && !receiveInfo.desktopServiceError) {
      visualStore.setReceiveInfo({
        userEmail: receiveInfo.email || receiveInfo.Email,
        serverUrl: receiveInfo.server || receiveInfo.Server,
@@ -337,6 +346,9 @@ export async function processMatrixView(
        projectId: receiveInfo.projectId || receiveInfo.ProjectId
      })
      console.log(`Receive info retrieved from desktop service - credentials loaded`)
+    } else {
+      desktopServiceUnavailable = true
+      console.log('Desktop service unavailable - cannot retrieve credentials')
    }

    // Now get the data from visual store for Speckle API download
@@ -345,9 +357,15 @@ export async function processMatrixView(
    const projectId = visualStore.receiveInfo?.projectId

    if (!token || !serverUrl || !projectId) {
-      visualStore.setCommonError(
-        'Missing Speckle credentials. Please refresh the data from the data connector.'
-      )
+      if (desktopServiceUnavailable) {
+        visualStore.setCommonError(
+          'Speckle Desktop Service is not running. Please start Speckle Desktop Services and refresh data.'
+        )
+      } else {
+        visualStore.setCommonError(
+          'Missing Speckle credentials. Please refresh the data from the data connector.'
+        )
+      }
      visualStore.setViewerReadyToLoad(false)
      return {
        modelObjects: [],
@@ -95,7 +95,6 @@ export class Visual implements IVisual {
    )

    visualStore.setFormattingSettings(this.formattingSettings)
-    console.log('Selector colors', this.formattingSettings.colorSelector)
    console.log(
      'Data Loading - Internalize Data:',
      this.formattingSettings.dataLoading.internalizeData.value
@@ -314,69 +313,7 @@ export class Visual implements IVisual {

  public getFormattingModel(): powerbi.visuals.FormattingModel {
    console.log('🎨 getFormattingModel called')
-
-    // build the cards for the options
-    const model: powerbi.visuals.FormattingModel = {
-      cards: [
-        // Color card
-        {
-          displayName: 'Object Display',
-          name: 'color',
-          uid: 'color_card_uid',
-          groups: [
-            {
-              displayName: undefined,
-              uid: 'color_group_uid',
-              slices: [
-                {
-                  displayName: 'Enabled',
-                  uid: 'color_enabled_uid',
-                  control: {
-                    type: powerbi.visuals.FormattingComponent.ToggleSwitch,
-                    properties: {
-                      descriptor: {
-                        objectName: 'color',
-                        propertyName: 'enabled'
-                      },
-                      value: this.formattingSettings.color.enabled.value
-                    }
-                  }
-                }
-              ]
-            }
-          ]
-        },
-        // Data Management card
-        {
-          displayName: 'Data Management',
-          name: 'dataLoading',
-          uid: 'dataLoading_card_uid',
-          groups: [
-            {
-              displayName: undefined,
-              uid: 'dataLoading_group_uid',
-              slices: [
-                {
-                  displayName: 'Internalize Data',
-                  uid: 'dataLoading_internalizeData_uid',
-                  control: {
-                    type: powerbi.visuals.FormattingComponent.ToggleSwitch,
-                    properties: {
-                      descriptor: {
-                        objectName: 'dataLoading',
-                        propertyName: 'internalizeData'
-                      },
-                      value: this.formattingSettings.dataLoading.internalizeData.value
-                    }
-                  }
-                }
-              ]
-            }
-          ]
-        }
-      ]
-    }
-
+    const model = this.formattingSettingsService.buildFormattingModel(this.formattingSettings)
    return model
  }
Author	SHA1	Message	Date
Dogukan Karatas	78af91f38a	Merge pull request #202 from specklesystems/dogukan/cnx-2515-fallback-to-json-for-scheduled-refresh Build and deploy Connector and Visual / build-connector (push) Has been cancelled Details Build and deploy Connector and Visual / build-visual (push) Has been cancelled Details Build and deploy Connector and Visual / deploy-installers (push) Has been cancelled Details fallback to json download	2025-09-11 15:02:39 +02:00
Dogukan Karatas	108a406bd5	clearer error message on visual	2025-09-11 15:00:00 +02:00
Mucahit Bilal GOKER	d7ede2edcf	Merge branch 'main' into dogukan/cnx-2515-fallback-to-json-for-scheduled-refresh	2025-09-11 07:13:22 +03:00
Mucahit Bilal GOKER	a25d635ca1	Merge pull request #201 from specklesystems/bilal/cnx-2510-remove-internal-border Bilal/cnx 2510 remove internal border	2025-09-11 06:57:59 +03:00
Dogukan Karatas	5a9add6d76	fallback to json download	2025-09-10 13:22:41 +02:00
Mucahit Bilal GOKER	89c8005dee	remove border from main container	2025-09-09 19:19:21 +03:00
Mucahit Bilal GOKER	a384370652	Merge branch 'dev' into bilal/cnx-2510-remove-internal-border	2025-09-09 19:07:56 +03:00
oguzhankoral	5ec90095f0	Merge branch 'dev' # Conflicts: # README.md	2025-09-04 23:29:20 +03:00
Oğuzhan Koral	20fad26fef	Merge pull request #200 from specklesystems/oguzhan/cherry-pick-readme Update README.md (#147)	2025-09-04 23:19:32 +03:00
Jonathon Broughton	03215f79c4	Update README.md (#147 ) * Update README.md * Update README.md (cherry picked from commit `85f8f72335`) # Conflicts: # README.md	2025-09-04 23:15:19 +03:00
Dogukan Karatas	6d17377ca2	Merge pull request #198 from specklesystems/dogukan/remove-access-token-auth Build and deploy Connector and Visual / build-connector (push) Has been cancelled Details Build and deploy Connector and Visual / build-visual (push) Has been cancelled Details Build and deploy Connector and Visual / deploy-installers (push) Has been cancelled Details feat (data): remove token and anonymous authentication	2025-09-03 10:09:43 +02:00
Dogukan Karatas	256abaed0c	remove token and anon auth	2025-09-02 21:46:35 +02:00
Dogukan Karatas	26409b4ea6	storing logic changes	2025-09-02 14:22:03 +02:00
Dogukan Karatas	865c4c1608	Merge pull request #197 from specklesystems/dogukan/server-url-in-request-data fix (data): adds server url to exchange data	2025-09-02 12:53:44 +02:00
Dogukan Karatas	67836c2a7f	case sensitive change	2025-09-02 11:00:35 +02:00
Dogukan Karatas	95d819f7f3	adds server url to exchange request data	2025-09-02 10:31:01 +02:00
Dogukan Karatas	dee3ee6c4d	Merge pull request #196 from specklesystems/dogukan/fix-conditional-formatting-card Build and deploy Connector and Visual / build-connector (push) Has been cancelled Details Build and deploy Connector and Visual / build-visual (push) Has been cancelled Details Build and deploy Connector and Visual / deploy-installers (push) Has been cancelled Details fix (visual): bring back the conditional formatting	2025-08-28 17:17:27 +02:00
Dogukan Karatas	7ed612ec14	revert back for conditional formatting	2025-08-28 16:30:15 +02:00
Jonathon Broughton	85f8f72335	Update README.md (#147 ) * Update README.md * Update README.md	2025-04-08 21:32:04 +03:00