Issue with s3 Data Connection Tutorial2

yuhyunch · January 7, 2025, 8:02am

Hi I’m following the Deep Dive: Creating Your First Data Connection. After following the tutorial I get Configuration error with below error message. I have added egress policy with s3-bucket.sandbox.training.palantir.com.s3.eu-west-1.amazonaws.com (Port 443). However, it is not working. Can someone help me with this?

The explorer command failed to run:
java.lang.Throwable:RemoteException: CUSTOM_CLIENT (ExplorerCommand:ExplorerCommandFailed) with instance ID cb19edac-312a-414a-8ad3-3cb33c847034: {stacktrace=com.palantir.magritte.plugin.s3.errors.AwsForbiddenEnhancement.enhanceException(AwsForbiddenEnhancement.java:30)
com.palantir.magritte.plugin.s3.errors.AwsExceptionEnhancer.lambda$enhanceException$0(AwsExceptionEnhancer.java:46)
java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:273)
java.util.AbstractList$RandomAccessSpliterator.tryAdvance(AbstractList.java:706)
java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150)
java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647)
com.palantir.magritte.plugin.s3.errors.AwsExceptionEnhancer.enhanceException(AwsExceptionEnhancer.java:47)
com.palantir.magritte.plugin.s3.errors.AwsExceptionEnhancer.runWithExceptionEnhancement(AwsExceptionEnhancer.java:37)
com.palantir.magritte.plugin.s3.WrappedS3Client.listObjectsV2(WrappedS3Client.java:37)
com.palantir.magritte.plugin.s3.S3Crawler$ListObjectsV2ResultIterator.computeNext(S3Crawler.java:49)
com.palantir.magritte.plugin.s3.S3Crawler$ListObjectsV2ResultIterator.computeNext(S3Crawler.java:30)
com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:145)
com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:140)
java.util.Iterator.forEachRemaining(Iterator.java:132)
java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1845)
java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:682)
com.palantir.magritte.plugin.s3.S3FileAndDirExplorer.getFileTreeNodes(S3FileAndDirExplorer.java:44)
com.palantir.magritte.plugin.s3.S3BucketReader.getFileTreeNodes(S3BucketReader.java:68)
com.palantir.magritte.plugin.s3.S3DirectSource.getFileTreeNodes(S3DirectSource.java:265)
com.palantir.magritte.source.explore.FileBasedExplorableSource$1.visitGetFileTreeNodesRequest(FileBasedExplorableSource.java:21)
com.palantir.magritte.source.explore.FileBasedExplorableSource$1.visitGetFileTreeNodesRequest(FileBasedExplorableSource.java:18)
com.palantir.magritte.explorer.api.FileBasedExplorationRequest$GetFileTreeNodesRequestWrapper.accept(FileBasedExplorationRequest.java:230)
com.palantir.magritte.explorer.api.FileBasedExplorationRequest.accept(FileBasedExplorationRequest.java:65)
com.palantir.magritte.source.explore.FileBasedExplorableSource.exploreFileBased(FileBasedExplorableSource.java:18)
com.palantir.magritte.api.Source$1.visitFileBased(Source.java:40)
com.palantir.magritte.api.Source$1.visitFileBased(Source.java:37)
com.palantir.magritte.explorer.api.ExplorationRequest$FileBasedWrapper.accept(ExplorationRequest.java:315)
com.palantir.magritte.explorer.api.ExplorationRequest.accept(ExplorationRequest.java:86)
com.palantir.magritte.api.Source.explore(Source.java:37)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationResource.lambda$getExplorationResponse$16(CloudSourceExplorationResource.java:174)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationResource.rethrowRuntimeExceptionsAsExplorerCommandFailures(CloudSourceExplorationResource.java:221)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationResource.getExplorationResponse(CloudSourceExplorationResource.java:159)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationServiceEndpoints$GetExplorationResponseEndpoint.handleRequest(CloudSourceExplorationServiceEndpoints.java:68)
com.palantir.conjure.java.undertow.runtime.ConjureExceptionHandler.handleRequest(ConjureExceptionHandler.java:42)
com.palantir.tracing.undertow.TracedStateHandler.handleRequest(TracedStateHandler.java:44)
com.palantir.conjure.java.undertow.runtime.LoggingContextHandler.handleRequest(LoggingContextHandler.java:40)
io.undertow.server.Connectors.executeRootHandler(Connectors.java:395)
io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:861)
com.palantir.witchcraft.ActiveCountingExecutorService$TaskWrapper.run(ActiveCountingExecutorService.java:84)
com.palantir.nylon.threads.RenamingExecutorService$RenamingRunnable.run(RenamingExecutorService.java:92)
org.jboss.threads.EnhancedViewExecutor$EnhancedViewExecutorRunnable.run(EnhancedViewExecutor.java:501)
org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
org.jboss.threads.EnhancedQueueExecutor$Task.doRunWith(EnhancedQueueExecutor.java:2675)
org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2654)
org.jboss.threads.EnhancedQueueExecutor.runThreadBody(EnhancedQueueExecutor.java:1627)
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1594)
com.palantir.tritium.metrics.TaggedMetricsThreadFactory$InstrumentedTask.run(TaggedMetricsThreadFactory.java:94)
java.lang.Thread.run(Thread.java:840), exceptionClass=com.palantir.logsafe.exceptions.SafeRuntimeException, message=The credentials provided have insufficient rights to access the requested resource, please check that the provided credentials are valid and have the required permissions. This might be caused by the bucket not allowing request from the job's origin. For agents, you will need to allow incoming traffic from the agent host or from the proxy if you have one setup (this can be setup in the source config or in the agent advanced config via JVM properties). For direct connection, if your bucket is in the same region as the Foundry stack please allow incoming traffic from the Foundry vpc endpoint (please contact Palantir to get the right endpoint). If your bucket is in a different region please allow incoming requests from the Foundry egress IPs found in the control panel app.: {exceptionClass=com.amazonaws.services.s3.model.AmazonS3Exception, isRetryable=true, statusCode=403, errorType=Client, serviceName=Amazon S3, errorCode=AccessDenied, requestId=XX295S7KYGV6AJVD, errorMessage=User: arn:aws:iam::441477072185:user/s3-training-test-user-1 is not authorized to perform: s3:ListBucket on resource: "arn:aws:s3:::s3-bucket.sandbox.training.palantir.com" because no identity-based policy allows the s3:ListBucket action (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: XX295S7KYGV6AJVD; S3 Extended Request ID: 9grn1WPXTH5bLH2KbE34p7lnfUTdifupAmfDkZ4Go5GFJmzHu6yr6/HEXB1bHxQ6p+vLFPzeThw=; Proxy: )}}
com.palantir.conjure.java.dialogue.serde.DefaultClients.newRemoteException(DefaultClients.java:148)
com.palantir.conjure.java.dialogue.serde.DefaultClients.block(DefaultClients.java:123)
com.palantir.conjure.java.dialogue.serde.DefaultClients.callBlocking(DefaultClients.java:76)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationServiceBlocking$1.getExplorationResponse(CloudSourceExplorationServiceBlocking.java:84)
com.palantir.magritte.coordinator.command.CloudSourceExplorationRequestMapper$1.subtype(CloudSourceExplorationRequestMapper.java:102)
com.palantir.magritte.coordinator.command.CloudSourceExplorationRequestMapper$1.subtype(CloudSourceExplorationRequestMapper.java:62)
com.palantir.magritte.bridge.command.SourceExplorationCommand.map(SourceExplorationCommand.java:50)
com.palantir.magritte.coordinator.command.CloudSourceExplorationRequestMapper.getCloudSourceExplorationResponse(CloudSourceExplorationRequestMapper.java:62)
com.palantir.magritte.coordinator.command.SourceExplorerCommandRunner.getCloudRunExplorerResponse(SourceExplorerCommandRunner.java:144)
com.palantir.magritte.coordinator.command.SourceExplorerCommandRunner.lambda$getExplorerResponseForSourceWithReadyRuntime$1(SourceExplorerCommandRunner.java:117)
com.palantir.magritte.store.source.api.RuntimePlatformResponse$VisitorBuilder$1.visitCloud(RuntimePlatformResponse.java:175)
com.palantir.magritte.store.source.api.RuntimePlatformResponse$CloudWrapper.accept(RuntimePlatformResponse.java:297)
com.palantir.magritte.store.source.api.RuntimePlatformResponse.accept(RuntimePlatformResponse.java:70)
com.palantir.magritte.coordinator.command.SourceExplorerCommandRunner.getExplorerResponseForSourceWithReadyRuntime(SourceExplorerCommandRunner.java:101)
com.palantir.magritte.coordinator.command.SourceExplorerCommandRunner.getExplorerResponseForSource(SourceExplorerCommandRunner.java:92)
com.palantir.magritte.coordinator.resources.FileBasedSourceExplorationResource.getFileTreeNodes(FileBasedSourceExplorationResource.java:51)
com.palantir.magritte.coordinator.api.FileBasedSourceExplorationServiceEndpoints$GetFileTreeNodesEndpoint.handleRequest(FileBasedSourceExplorationServiceEndpoints.java:73)
com.palantir.conjure.java.undertow.runtime.ConjureExceptionHandler.handleRequest(ConjureExceptionHandler.java:42)
com.palantir.tracing.undertow.TracedStateHandler.handleRequest(TracedStateHandler.java:44)
com.palantir.conjure.java.undertow.runtime.LoggingContextHandler.handleRequest(LoggingContextHandler.java:40)
io.undertow.server.Connectors.executeRootHandler(Connectors.java:395)
io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:861)
com.palantir.witchcraft.ActiveCountingExecutorService$TaskWrapper.run(ActiveCountingExecutorService.java:84)
com.palantir.nylon.threads.RenamingExecutorService$RenamingRunnable.run(RenamingExecutorService.java:92)
org.jboss.threads.EnhancedViewExecutor$EnhancedViewExecutorRunnable.run(EnhancedViewExecutor.java:501)
org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
org.jboss.threads.EnhancedQueueExecutor$Task.doRunWith(EnhancedQueueExecutor.java:2675)
org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2654)
org.jboss.threads.EnhancedQueueExecutor.runThreadBody(EnhancedQueueExecutor.java:1627)
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1594)
com.palantir.tritium.metrics.TaggedMetricsThreadFactory$InstrumentedTask.run(TaggedMetricsThreadFactory.java:94)
java.lang.Thread.run(Thread.java:840)

tpm · January 8, 2025, 2:12pm

Hi,

Sorry to hear you are having issues with the tutorial. We have identified and fixed a problem that could be causing this, could you please try again and let us know if it works now?

Thank you,

Tomasz

yuhyunch · January 13, 2025, 9:04am

It worked! Thank you so much. But having the similar issue for the Data Connection Training - Postgres. Could you also take a look at this?

The explorer command failed to run:
java.lang.Throwable:RemoteException: INVALID_ARGUMENT (MagritteExplorer:SqlError) with instance ID 610bbbe9-bfcf-4f45-9ce7-03a3b2b08ea8: {reason=Connection to [REDACTED URL]:5432 refused. Check that the hostname and port are correct and that the postmaster is accepting TCP/IP connections., originalErrorInstanceId=17bbbe4a-a67e-4276-8f37-b44c1bd8cf7a, stacktrace=com.palantir.magritte.explorer.jdbc.MagritteExplorerErrors.sqlError(MagritteExplorerErrors.java:25)
shadow.com.palantir.magritte.magritte_jdbc_postgresql_42_source.com.palantir.magritte.jdbc.explorer.JdbcExplorationRunner.runExplorationRequest(JdbcExplorationRunner.java:164)
shadow.com.palantir.magritte.magritte_jdbc_postgresql_42_source.com.palantir.magritte.jdbc.explorer.JdbcExplorationRunner.getTables(JdbcExplorationRunner.java:65)
com.palantir.magritte.plugin.jdbc.postgresql42.Postgresql42Source.getTables(Postgresql42Source.java:68)
com.palantir.magritte.source.explore.TableBasedExplorableSourceV2$1.visitGetTablesRequest(TableBasedExplorableSourceV2.java:30)
com.palantir.magritte.source.explore.TableBasedExplorableSourceV2$1.visitGetTablesRequest(TableBasedExplorableSourceV2.java:27)
com.palantir.magritte.explorer.api.TableBasedExplorationRequestV2$GetTablesRequestWrapper.accept(TableBasedExplorationRequestV2.java:326)
com.palantir.magritte.explorer.api.TableBasedExplorationRequestV2.accept(TableBasedExplorationRequestV2.java:90)
com.palantir.magritte.source.explore.TableBasedExplorableSourceV2.exploreTableBasedV2(TableBasedExplorableSourceV2.java:27)
com.palantir.magritte.api.Source$1.visitTableBasedV2(Source.java:50)
com.palantir.magritte.api.Source$1.visitTableBasedV2(Source.java:37)
com.palantir.magritte.explorer.api.ExplorationRequest$TableBasedV2Wrapper.accept(ExplorationRequest.java:405)
com.palantir.magritte.explorer.api.ExplorationRequest.accept(ExplorationRequest.java:86)
com.palantir.magritte.api.Source.explore(Source.java:37)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationResource.lambda$getExplorationResponse$16(CloudSourceExplorationResource.java:174)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationResource.rethrowRuntimeExceptionsAsExplorerCommandFailures(CloudSourceExplorationResource.java:221)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationResource.getExplorationResponse(CloudSourceExplorationResource.java:159)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationServiceEndpoints$GetExplorationResponseEndpoint.handleRequest(CloudSourceExplorationServiceEndpoints.java:68)
com.palantir.conjure.java.undertow.runtime.ConjureExceptionHandler.handleRequest(ConjureExceptionHandler.java:42)
com.palantir.tracing.undertow.TracedStateHandler.handleRequest(TracedStateHandler.java:44)
com.palantir.conjure.java.undertow.runtime.LoggingContextHandler.handleRequest(LoggingContextHandler.java:40)
io.undertow.server.Connectors.executeRootHandler(Connectors.java:395)
io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:861)
com.palantir.witchcraft.ActiveCountingExecutorService$TaskWrapper.run(ActiveCountingExecutorService.java:84)
com.palantir.nylon.threads.RenamingExecutorService$RenamingRunnable.run(RenamingExecutorService.java:92)
org.jboss.threads.EnhancedViewExecutor$EnhancedViewExecutorRunnable.run(EnhancedViewExecutor.java:501)
org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
org.jboss.threads.EnhancedQueueExecutor$Task.doRunWith(EnhancedQueueExecutor.java:2675)
org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2654)
org.jboss.threads.EnhancedQueueExecutor.runThreadBody(EnhancedQueueExecutor.java:1627)
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1594)
com.palantir.tritium.metrics.TaggedMetricsThreadFactory$InstrumentedTask.run(TaggedMetricsThreadFactory.java:94)
java.lang.Thread.run(Thread.java:840), sqlState=08001, vendorCode=0}
com.palantir.conjure.java.dialogue.serde.DefaultClients.newRemoteException(DefaultClients.java:148)
com.palantir.conjure.java.dialogue.serde.DefaultClients.block(DefaultClients.java:123)
com.palantir.conjure.java.dialogue.serde.DefaultClients.callBlocking(DefaultClients.java:76)
com.palantir.magritte.cloud.explorer.CloudSourceExplorationServiceBlocking$1.getExplorationResponse(CloudSourceExplorationServiceBlocking.java:84)
com.palantir.magritte.coordinator.command.CloudSourceExplorationRequestMapper$1.subtype(CloudSourceExplorationRequestMapper.java:102)
com.palantir.magritte.coordinator.command.CloudSourceExplorationRequestMapper$1.subtype(CloudSourceExplorationRequestMapper.java:62)
com.palantir.magritte.bridge.command.SourceExplorationCommand.map(SourceExplorationCommand.java:50)
com.palantir.magritte.coordinator.command.CloudSourceExplorationRequestMapper.getCloudSourceExplorationResponse(CloudSourceExplorationRequestMapper.java:62)
com.palantir.magritte.coordinator.command.SourceExplorerCommandRunner.getCloudRunExplorerResponse(SourceExplorerCommandRunner.java:144)
com.palantir.magritte.coordinator.command.SourceExplorerCommandRunner.lambda$getExplorerResponseForSourceWithReadyRuntime$1(SourceExplorerCommandRunner.java:117)
com.palantir.magritte.store.source.api.RuntimePlatformResponse$VisitorBuilder$1.visitCloud(RuntimePlatformResponse.java:175)
com.palantir.magritte.store.source.api.RuntimePlatformResponse$CloudWrapper.accept(RuntimePlatformResponse.java:297)
com.palantir.magritte.store.source.api.RuntimePlatformResponse.accept(RuntimePlatformResponse.java:70)
com.palantir.magritte.coordinator.command.SourceExplorerCommandRunner.getExplorerResponseForSourceWithReadyRuntime(SourceExplorerCommandRunner.java:101)
com.palantir.magritte.coordinator.command.SourceExplorerCommandRunner.getExplorerResponseForSource(SourceExplorerCommandRunner.java:92)
com.palantir.magritte.coordinator.resources.TableBasedSourceExplorationResource.getExplorerResponse(TableBasedSourceExplorationResource.java:202)
com.palantir.magritte.coordinator.resources.TableBasedSourceExplorationResource.getTables(TableBasedSourceExplorationResource.java:78)
com.palantir.magritte.coordinator.api.TableBasedSourceExplorationServiceEndpoints$GetTablesEndpoint.handleRequest(TableBasedSourceExplorationServiceEndpoints.java:81)
com.palantir.conjure.java.undertow.runtime.ConjureExceptionHandler.handleRequest(ConjureExceptionHandler.java:42)
com.palantir.tracing.undertow.TracedStateHandler.handleRequest(TracedStateHandler.java:44)
com.palantir.conjure.java.undertow.runtime.LoggingContextHandler.handleRequest(LoggingContextHandler.java:40)
io.undertow.server.Connectors.executeRootHandler(Connectors.java:395)
io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:861)
com.palantir.witchcraft.ActiveCountingExecutorService$TaskWrapper.run(ActiveCountingExecutorService.java:84)
com.palantir.nylon.threads.RenamingExecutorService$RenamingRunnable.run(RenamingExecutorService.java:92)
org.jboss.threads.EnhancedViewExecutor$EnhancedViewExecutorRunnable.run(EnhancedViewExecutor.java:501)
org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18)
org.jboss.threads.EnhancedQueueExecutor$Task.doRunWith(EnhancedQueueExecutor.java:2675)
org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2654)
org.jboss.threads.EnhancedQueueExecutor.runThreadBody(EnhancedQueueExecutor.java:1627)
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1594)
com.palantir.tritium.metrics.TaggedMetricsThreadFactory$InstrumentedTask.run(TaggedMetricsThreadFactory.java:94)
java.lang.Thread.run(Thread.java:840)