- Is this your first time deploying Airbyte?: No
- Deployment: Kubernetes
- Airbyte Version: 0.39.42-alpha
- Description:
I am trying to add a custom source/destination through the UI, I can’t get past get spec job failed
My image is on a public container registry, I can run docker run image_name spec, the worker is able to start a pod to create the destination which completes. Theres not much I can get out of these error logs. Would appreciate any hints in where to look. Thanks!
Server Logs:
2022-08-09 02:36:37 ERROR i.a.s.e.UncaughtExceptionMapper(toResponse):22 - Uncaught exception
java.lang.IllegalStateException: Get Spec job failed.
at com.google.common.base.Preconditions.checkState(Preconditions.java:502) ~[guava-31.0.1-jre.jar:?]
at io.airbyte.server.converters.SpecFetcher.getSpecFromJob(SpecFetcher.java:14) ~[io.airbyte-airbyte-server-0.39.42-alpha.jar:?]
at io.airbyte.server.handlers.DestinationDefinitionsHandler.getSpecForImage(DestinationDefinitionsHandler.java:290) ~[io.airbyte-airbyte-server-0.39.42-alpha.jar:?]
at io.airbyte.server.handlers.DestinationDefinitionsHandler.destinationDefinitionFromCreate(DestinationDefinitionsHandler.java:195) ~[io.airbyte-airbyte-server-0.39.42-alpha.jar:?]
at io.airbyte.server.handlers.DestinationDefinitionsHandler.createPrivateDestinationDefinition(DestinationDefinitionsHandler.java:175) ~[io.airbyte-airbyte-server-0.39.42-alpha.jar:?]
at io.airbyte.server.apis.ConfigurationApi.lambda$createDestinationDefinition$43(ConfigurationApi.java:542) ~[io.airbyte-airbyte-server-0.39.42-alpha.jar:?]
at io.airbyte.server.apis.ConfigurationApi.execute(ConfigurationApi.java:873) ~[io.airbyte-airbyte-server-0.39.42-alpha.jar:?]
at io.airbyte.server.apis.ConfigurationApi.createDestinationDefinition(ConfigurationApi.java:542) ~[io.airbyte-airbyte-server-0.39.42-alpha.jar:?]
at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:578) ~[?:?]
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:52) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:124) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:167) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$TypeOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:219) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:79) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:469) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:391) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:80) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:253) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) ~[jersey-common-2.31.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) ~[jersey-common-2.31.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:292) ~[jersey-common-2.31.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:274) ~[jersey-common-2.31.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:244) ~[jersey-common-2.31.jar:?]
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) ~[jersey-common-2.31.jar:?]
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:232) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:680) ~[jersey-server-2.31.jar:?]
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:394) ~[jersey-container-servlet-core-2.31.jar:?]
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:346) ~[jersey-container-servlet-core-2.31.jar:?]
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:366) ~[jersey-container-servlet-core-2.31.jar:?]
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:319) ~[jersey-container-servlet-core-2.31.jar:?]
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:205) ~[jersey-container-servlet-core-2.31.jar:?]
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:763) ~[jetty-servlet-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:569) ~[jetty-servlet-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) ~[jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1377) ~[jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) ~[jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:507) ~[jetty-servlet-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) ~[jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1292) ~[jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) ~[jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) ~[jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.Server.handle(Server.java:501) ~[jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) ~[jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:556) [jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) [jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273) [jetty-server-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) [jetty-io-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) [jetty-io-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) [jetty-io-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336) [jetty-util-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313) [jetty-util-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) [jetty-util-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129) [jetty-util-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:375) [jetty-util-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) [jetty-util-9.4.31.v20200723.jar:9.4.31.v20200723]
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) [jetty-util-9.4.31.v20200723.jar:9.4.31.v20200723]
at java.lang.Thread.run(Thread.java:1589) [?:?]
2022-08-09 02:36:37 ERROR i.a.s.RequestLogger(filter):110 - REQ 127.0.0.6 POST 500 /api/v1/destination_definitions/create - {"name":"CSS Kafka","documentationUrl":"localhost","dockerImageTag":"dev","dockerRepository":"gcr.io/css-data-sandbox/airbyte/destination-css-kafka"}
Worker Logs:
2022-08-09 17:51:01 ERROR i.t.i.s.WorkflowExecuteRunnable(logWorkflowExecutionException):125 - Workflow execution failure WorkflowId=979faf4b-f3f7-498e-a973-34f9b44d34b9, RunId=375d32d7-fcac-4cfd-a9d3-ef738055a7a6, WorkflowType=SpecWorkflow
io.temporal.failure.ActivityFailure: scheduledEventId=5, startedEventId=6, activityType='Run', activityId='a7b86cea-d59b-369e-8975-06d05be34c2f', identity='1@airbyte-worker-698f4f4c6d-p6bpg', retryState=RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED
at java.lang.Thread.getStackTrace(Thread.java:2550) ~[?:?]
at io.temporal.internal.sync.ActivityStubBase.execute(ActivityStubBase.java:48) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.sync.ActivityInvocationHandler.lambda$getActivityFunc$0(ActivityInvocationHandler.java:77) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.sync.ActivityInvocationHandlerBase.invoke(ActivityInvocationHandlerBase.java:70) ~[temporal-sdk-1.8.1.jar:?]
at jdk.proxy2.$Proxy44.run(Unknown Source) ~[?:?]
at io.airbyte.workers.temporal.spec.SpecWorkflowImpl.run(SpecWorkflowImpl.java:25) ~[io.airbyte-airbyte-workers-0.39.42-alpha.jar:?]
at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:578) ~[?:?]
at io.temporal.internal.sync.POJOWorkflowImplementationFactory$POJOWorkflowImplementation$RootWorkflowInboundCallsInterceptor.execute(POJOWorkflowImplementationFactory.java:317) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.sync.POJOWorkflowImplementationFactory$POJOWorkflowImplementation.execute(POJOWorkflowImplementationFactory.java:292) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.sync.WorkflowExecuteRunnable.run(WorkflowExecuteRunnable.java:72) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.sync.SyncWorkflow.lambda$start$0(SyncWorkflow.java:137) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.sync.CancellationScopeImpl.run(CancellationScopeImpl.java:101) [temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.sync.WorkflowThreadImpl$RunnableWrapper.run(WorkflowThreadImpl.java:111) [temporal-sdk-1.8.1.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:577) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:317) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) [?:?]
at java.lang.Thread.run(Thread.java:1589) [?:?]
Caused by: io.temporal.failure.ApplicationFailure: message='io.airbyte.workers.exception.WorkerException: Error while getting spec from image gcr.io/css-data-sandbox/airbyte/destination-css-kafka:dev', type='java.util.concurrent.ExecutionException', nonRetryable=false
at java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
at io.airbyte.workers.temporal.TemporalAttemptExecution.get(TemporalAttemptExecution.java:129) ~[io.airbyte-airbyte-workers-0.39.42-alpha.jar:?]
at io.airbyte.workers.temporal.spec.SpecActivityImpl.run(SpecActivityImpl.java:72) ~[io.airbyte-airbyte-workers-0.39.42-alpha.jar:?]
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104) ~[?:?]
at java.base/java.lang.reflect.Method.invoke(Method.java:578) ~[?:?]
at io.temporal.internal.activity.POJOActivityTaskHandler$POJOActivityInboundCallsInterceptor.execute(POJOActivityTaskHandler.java:214) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.activity.POJOActivityTaskHandler$POJOActivityImplementation.execute(POJOActivityTaskHandler.java:180) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.activity.POJOActivityTaskHandler.handle(POJOActivityTaskHandler.java:120) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.worker.ActivityWorker$TaskHandlerImpl.handle(ActivityWorker.java:204) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.worker.ActivityWorker$TaskHandlerImpl.handle(ActivityWorker.java:164) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.worker.PollTaskExecutor.lambda$process$0(PollTaskExecutor.java:93) ~[temporal-sdk-1.8.1.jar:?]
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
at java.base/java.lang.Thread.run(Thread.java:1589) ~[?:?]
Caused by: io.temporal.failure.ApplicationFailure: message='Error while getting spec from image gcr.io/css-data-sandbox/airbyte/destination-css-kafka:dev', type='io.airbyte.workers.exception.WorkerException', nonRetryable=false
at io.airbyte.workers.general.DefaultGetSpecWorker.run(DefaultGetSpecWorker.java:92) ~[io.airbyte-airbyte-workers-0.39.42-alpha.jar:?]
at io.airbyte.workers.general.DefaultGetSpecWorker.run(DefaultGetSpecWorker.java:32) ~[io.airbyte-airbyte-workers-0.39.42-alpha.jar:?]
at io.airbyte.workers.temporal.TemporalAttemptExecution.lambda$getWorkerThread$2(TemporalAttemptExecution.java:155) ~[io.airbyte-airbyte-workers-0.39.42-alpha.jar:?]
at java.base/java.lang.Thread.run(Thread.java:1589) ~[?:?]
Caused by: io.temporal.failure.ApplicationFailure: type='java.lang.NullPointerException', nonRetryable=false
at java.base/java.io.Reader.<init>(Reader.java:168) ~[?:?]
at java.base/java.io.InputStreamReader.<init>(InputStreamReader.java:124) ~[?:?]
at io.airbyte.commons.io.IOs.newBufferedReader(IOs.java:119) ~[io.airbyte-airbyte-commons-0.39.42-alpha.jar:?]
at io.airbyte.commons.io.LineGobbler.<init>(LineGobbler.java:67) ~[io.airbyte-airbyte-commons-0.39.42-alpha.jar:?]
at io.airbyte.commons.io.LineGobbler.gobble(LineGobbler.java:35) ~[io.airbyte-airbyte-commons-0.39.42-alpha.jar:?]
at io.airbyte.commons.io.LineGobbler.gobble(LineGobbler.java:25) ~[io.airbyte-airbyte-commons-0.39.42-alpha.jar:?]
at io.airbyte.workers.general.DefaultGetSpecWorker.run(DefaultGetSpecWorker.java:59) ~[io.airbyte-airbyte-workers-0.39.42-alpha.jar:?]
at io.airbyte.workers.general.DefaultGetSpecWorker.run(DefaultGetSpecWorker.java:32) ~[io.airbyte-airbyte-workers-0.39.42-alpha.jar:?]
at io.airbyte.workers.temporal.TemporalAttemptExecution.lambda$getWorkerThread$2(TemporalAttemptExecution.java:155) ~[io.airbyte-airbyte-workers-0.39.42-alpha.jar:?]
at java.base/java.lang.Thread.run(Thread.java:1589) ~[?:?]
2022-08-09 17:51:04 INFO i.a.w.p.ExitCodeWatcher(persistExitCode):113 - Received exit code 0 for pod ion-css-kafka-spec-bcea2ce2-030a-4af4-a644-cc53e9a1043d-0-rinkg