Failed to fetch schema...!

Connector Development
1

Building a python source connector, Added it to airbyte GUI, setup source & destination is successful but failed to fetch schema
Ref:


import json
from datetime import datetime
from typing import Dict, Generator

from airbyte_cdk.logger import AirbyteLogger
from airbyte_cdk.models import (
    AirbyteCatalog,
    AirbyteConnectionStatus,
    AirbyteMessage,
    AirbyteRecordMessage,
    AirbyteStream,
    ConfiguredAirbyteCatalog,
    Status,
    Type,
)
from airbyte_cdk.sources import Source
from . import weather_list


class SourcePython(Source):
    url_base = "https://weatherapi-com.p.rapidapi.com/"

    def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus:

        try:
            input_list = config["q"]
            return AirbyteConnectionStatus(status=Status.SUCCEEDED)
        except Exception as e:
            return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}")

    def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog:
        streams = []

        stream_name = "sports"  # Example
        json_schema = {  # Example
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object",
            "properties": {
                "football": {
                    "type": "array",
                    "items": [
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        }
                    ]
                },
                "cricket": {
                    "type": "array",
                    "items": [
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        },
                        {
                            "type": "object",
                            "properties": {
                                "stadium": {
                                    "type": "string"
                                },
                                "country": {
                                    "type": "string"
                                },
                                "region": {
                                    "type": "string"
                                },
                                "tournament": {
                                    "type": "string"
                                },
                                "start": {
                                    "type": "string"
                                },
                                "match": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "stadium",
                                "country",
                                "region",
                                "tournament",
                                "start",
                                "match"
                            ]
                        }
                    ]
                },
                "golf": {
                    "type": "array",
                    "items": {}
                }
            },
        }
        streams.append(AirbyteStream(name=stream_name, json_schema=json_schema))
        return AirbyteCatalog(streams=streams)

    def read(
        self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any]
    ) -> Generator[AirbyteMessage, None, None]:

        stream_name = "sports"  # Example
        data = {"columnName": "football"}  # Example

        yield AirbyteMessage(
            type=Type.RECORD,
            record=AirbyteRecordMessage(stream=stream_name, data=data, emitted_at=int(datetime.now().timestamp()) * 1000),
        )
class sports(SourcePython):
    # Set this as a noop.
    primary_key = None
    def path(self, **kwargs) -> str:
        weather_list = self.weather_list
        return f"/sports.json?q={weather_list}&rapidapi-key=***************"
2

Hello @Bharath,
As your Sport stream schema looks static you should rather define it a dedicated JSON file as we suggest in our documentation: Defining Stream Schemas | Airbyte Documentation.

To debug the fetch schema failure I’d suggest your to run the discover locally using:
docker run --rm -v $(pwd)/secrets:/secrets airbyte/<your-connector-image-name>:dev discover --config /secrets/config.json
You should be able to get an insightful error after running this command.

4

I changed the schema it got succeded, but in transformations, if we keep raw JSON data it got succeded but we are unable to see the data in the destination
When we opt for Basic Normalization for sync, sync is failing with the error: Normalization Failed. Attaching the logs of the same.
logs-215.txt (25.1 KB)

5

Hi Bharath,
I think there’s still something wrong in your sport stream schema that leads to the generation of a bad catalog:

2022-04-11 06:21:57 e[42mnormalizatione[0m > KeyError: "'json_schema'.'properties' are not defined for stream sports"2022-04-11 06:21:57 e[42mnormalizatione[0m > 2022-04-11 06:21:57 e[42mnormalizatione[0m > Showing destination_catalog.json to diagnose/debug errors (1):2022-04-11 06:21:57 e[42mnormalizatione[0m > 2022-04-11 06:21:57 e[42mnormalizatione[0m > {2022-04-11 06:21:57 e[42mnormalizatione[0m > "streams": [2022-04-11 06:21:57 e[42mnormalizatione[0m > {2022-04-11 06:21:57 e[42mnormalizatione[0m > "stream": {2022-04-11 06:21:57 e[42mnormalizatione[0m > "name": "sports",2022-04-11 06:21:57 e[42mnormalizatione[0m > "json_schema": {},2022-04-11 06:21:57 e[42mnormalizatione[0m > "supported_sync_modes": [2022-04-11 06:21:57 e[42mnormalizatione[0m > "full_refresh"2022-04-11 06:21:57 e[42mnormalizatione[0m >],2022-04-11 06:21:57 e[42mnormalizatione[0m > "default_cursor_field": [],2022-04-11 06:21:57 e[42mnormalizatione[0m > "source_defined_primary_key": []2022-04-11 06:21:57 e[42mnormalizatione[0m > },2022-04-11 06:21:57 e[42mnormalizatione[0m > "sync_mode": "full_refresh",2022-04-11 06:21:57 e[42mnormalizatione[0m > "cursor_field": [],2022-04-11 06:21:57 e[42mnormalizatione[0m > "destination_sync_mode": "append",2022-04-11 06:21:57 e[42mnormalizatione[0m > "primary_key": []2022-04-11 06:21:57 e[42mnormalizatione[0m > }2022-04-11 06:21:57 e[42mnormalizatione[0m >]2022-04-11 06:21:57 e[42mnormalizatione[0m > }

Do you mind sharing your schema for the sports stream?

6

Hi there from the Community Assistance team.
We’re letting you know about an issue we discovered with the back-end process we use to handle topics and responses on the forum. If you experienced a situation where you posted the last message in a topic that did not receive any further replies, please open a new topic to continue the discussion. In addition, if you’re having a problem and find a closed topic on the subject, go ahead and open a new topic on it and we’ll follow up with you. We apologize for the inconvenience, and appreciate your willingness to work with us to provide a supportive community.