I’m creating a new connector and having what appear to be caching issues of my spec.json when I run the python source_acceptance_test suite. It seems like maybe docker is caching my spec and not updating it when I change it? Does anyone know how to force docker to not do any caching when I build my connector?
I noticed this from test_core which appears to suggest spec is being cached:
@pytest.fixture(name="actual_connector_spec")
def actual_connector_spec_fixture(request: BaseTest, docker_runner):
if not request.instance.spec_cache:
output = docker_runner.call_spec()
spec_messages = filter_output(output, Type.SPEC)
assert len(spec_messages) == 1, "Spec message should be emitted exactly once"
spec = spec_messages[0].spec
request.spec_cache = spec
return request.spec_cache
Here are the errors I’m getting:
self = <source_acceptance_test.tests.test_core.TestSpec object at 0x4008847490>
actual_connector_spec = ConnectorSpecification(documentationUrl=AnyUrl('https://docs.faros.ai', scheme='https', host='docs.faros.ai', tld='ai'...malization=False, supportsDBT=False, supported_destination_sync_modes=None, authSpecification=None, advanced_auth=None)
connector_config = SecretDict(******)
def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification, connector_config: SecretDict):
"""Check that config matches the actual schema from the spec call"""
# Getting rid of technical variables that start with an underscore
config = {key: value for key, value in connector_config.data.items() if not key.startswith("_")}
try:
jsonschema.validate(instance=config, schema=actual_connector_spec.connectionSpecification)
except jsonschema.exceptions.ValidationError as err:
> pytest.fail(f"Config invalid: {err}")
E Failed: Config invalid: 'test_url' is a required property
E
E Failed validating 'required' in schema:
E {'$schema': 'http://json-schema.org/draft-07/schema#',
E 'additionalProperties': False,
E 'properties': {'test_url': {'examples': ['https://my-jenkins-server.example.com'],
E 'title': 'Jenkins Server URL Test',
E 'type': 'string'},
E 'token': {'airbyte_secret': True,
E 'title': 'Jenkins Token',
E 'type': 'string'},
E 'user': {'title': 'Jenkins User', 'type': 'string'}},
E 'required': ['test_url', 'user', 'token'],
E 'title': 'Jenkins Spec',
E 'type': 'object'}
E
E On instance:
E {'server_url': 'https://eth-rinkeby.alchemyapi.io/v2/',
E 'token': '',
E 'user': 'chris'}
source_acceptance_test/tests/test_core.py:55: Failed
test_core.py ⨯ 5% ▌
―――――――――――――――――――― TestSpec.test_match_expected[inputs0] ―――――――――――――――――――――
self = <source_acceptance_test.tests.test_core.TestSpec object at 0x400a2dedc0>
connector_spec = ConnectorSpecification(documentationUrl=AnyUrl('https://docs.faros.ai', scheme='https', host='docs.faros.ai', tld='ai'...malization=False, supportsDBT=False, supported_destination_sync_modes=None, authSpecification=None, advanced_auth=None)
actual_connector_spec = ConnectorSpecification(documentationUrl=AnyUrl('https://docs.faros.ai', scheme='https', host='docs.faros.ai', tld='ai'...malization=False, supportsDBT=False, supported_destination_sync_modes=None, authSpecification=None, advanced_auth=None)
def test_match_expected(self, connector_spec: ConnectorSpecification, actual_connector_spec: ConnectorSpecification):
"""Check that spec call returns a spec equals to expected one"""
if connector_spec:
> assert actual_connector_spec == connector_spec, "Spec should be equal to the one in spec.yaml or spec.json file"
E AssertionError: Spec should be equal to the one in spec.yaml or spec.json file
E assert equals failed
E ConnectorSpecification(doc ConnectorSpecification(doc
E umentationUrl=AnyUrl('http umentationUrl=AnyUrl('http
E s://docs.faros.ai', scheme s://docs.faros.ai', scheme
E ='https', host='docs.faros ='https', host='docs.faros
E .ai', tld='ai', host_type= .ai', tld='ai', host_type=
E 'domain'), changelogUrl=No 'domain'), changelogUrl=No
E ne, connectionSpecificatio ne, connectionSpecificatio ...
E
E ...Full output truncated (27 lines hidden), use '-vv' to show
source_acceptance_test/tests/test_core.py:62: AssertionError
In the example above, my actual spec.json uses the key “server_url” and not “test_url”. “test_url” was something I had tried previously, but now appears cached and re-building my connector does not update it.