Incremental | Append mode is not working correctly for zendesk Sunshine> redshift connection

Connection: Zendesk Sunshine > Redshift
Stream selected for replication: Object Records
Primary key: id
Cursor Field: updated_at [source defined]

{
  "syncCatalog": {
    "streams": [
      {
        "config": {
          "syncMode": "full_refresh",
          "cursorField": [],
          "destinationSyncMode": "append",
          "primaryKey": [
            [
              "key"
            ]
          ],
          "aliasName": "limits",
          "selected": false
        },
        "stream": {
          "name": "limits",
          "jsonSchema": {
            "type": "object",
            "properties": {
              "key": {
                "type": "string"
              },
              "limit": {
                "type": "integer"
              },
              "count": {
                "type": "integer"
              }
            }
          },
          "supportedSyncModes": [
            "full_refresh"
          ],
          "sourceDefinedCursor": null,
          "defaultCursorField": [],
          "sourceDefinedPrimaryKey": [
            [
              "key"
            ]
          ],
          "namespace": null
        }
      },
      {
        "config": {
          "syncMode": "incremental",
          "cursorField": [
            "updated_at"
          ],
          "destinationSyncMode": "append",
          "primaryKey": [
            [
              "id"
            ]
          ],
          "aliasName": "object_records",
          "selected": true
        },
        "stream": {
          "name": "object_records",
          "jsonSchema": {
            "type": "object",
            "properties": {
              "type": {
                "type": "string"
              },
              "id": {
                "type": "string"
              },
              "external_id": {
                "type": [
                  "string",
                  "null"
                ]
              },
              "attributes": {
                "type": "object"
              },
              "created_at": {
                "type": "string"
              },
              "updated_at": {
                "type": "string"
              }
            }
          },
          "supportedSyncModes": [
            "full_refresh",
            "incremental"
          ],
          "sourceDefinedCursor": true,
          "defaultCursorField": [
            "updated_at"
          ],
          "sourceDefinedPrimaryKey": [
            [
              "id"
            ]
          ],
          "namespace": null
        }
      },
      {
        "config": {
          "syncMode": "full_refresh",
          "cursorField": [],
          "destinationSyncMode": "append",
          "primaryKey": [],
          "aliasName": "object_type_policies",
          "selected": false
        },
        "stream": {
          "name": "object_type_policies",
          "jsonSchema": {
            "type": "object",
            "properties": {
              "object_type": {
                "type": "string"
              },
              "rbac": {
                "type": "object",
                "properties": {
                  "admin": {
                    "type": "object",
                    "properties": {
                      "create": {
                        "type": "boolean"
                      },
                      "read": {
                        "type": "boolean"
                      },
                      "update": {
                        "type": "boolean"
                      },
                      "delete": {
                        "type": "boolean"
                      }
                    }
                  },
                  "agent": {
                    "type": "object",
                    "properties": {
                      "create": {
                        "type": "boolean"
                      },
                      "read": {
                        "type": "boolean"
                      },
                      "update": {
                        "type": "boolean"
                      },
                      "delete": {
                        "type": "boolean"
                      }
                    }
                  },
                  "end_user": {
                    "type": "object",
                    "properties": {
                      "create": {
                        "type": "boolean"
                      },
                      "read": {
                        "type": "boolean"
                      },
                      "update": {
                        "type": "boolean"
                      },
                      "delete": {
                        "type": "boolean"
                      }
                    }
                  }
                }
              }
            }
          },
          "supportedSyncModes": [
            "full_refresh"
          ],
          "sourceDefinedCursor": null,
          "defaultCursorField": [],
          "sourceDefinedPrimaryKey": [],
          "namespace": null
        }
      },
      {
        "config": {
          "syncMode": "full_refresh",
          "cursorField": [],
          "destinationSyncMode": "append",
          "primaryKey": [
            [
              "key"
            ]
          ],
          "aliasName": "object_types",
          "selected": false
        },
        "stream": {
          "name": "object_types",
          "jsonSchema": {
            "type": "object",
            "properties": {
              "key": {
                "type": "string"
              },
              "schema": {
                "type": "object",
                "properties": {
                  "properties": {
                    "type": "object"
                  },
                  "required": {
                    "type": "array",
                    "items": {
                      "type": "string"
                    }
                  },
                  "additionalProperties": {
                    "type": "boolean"
                  }
                }
              },
              "created_at": {
                "type": "string"
              },
              "updated_at": {
                "type": "string"
              }
            }
          },
          "supportedSyncModes": [
            "full_refresh"
          ],
          "sourceDefinedCursor": null,
          "defaultCursorField": [],
          "sourceDefinedPrimaryKey": [
            [
              "key"
            ]
          ],
          "namespace": null
        }
      },
      {
        "config": {
          "syncMode": "full_refresh",
          "cursorField": [],
          "destinationSyncMode": "append",
          "primaryKey": [
            [
              "id"
            ]
          ],
          "aliasName": "relationship_records",
          "selected": false
        },
        "stream": {
          "name": "relationship_records",
          "jsonSchema": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "relationship_type": {
                "type": "string"
              },
              "source": {
                "type": "string"
              },
              "target": {
                "type": "string"
              },
              "created_at": {
                "type": "string"
              }
            }
          },
          "supportedSyncModes": [
            "full_refresh"
          ],
          "sourceDefinedCursor": null,
          "defaultCursorField": [],
          "sourceDefinedPrimaryKey": [
            [
              "id"
            ]
          ],
          "namespace": null
        }
      },
      {
        "config": {
          "syncMode": "full_refresh",
          "cursorField": [],
          "destinationSyncMode": "append",
          "primaryKey": [
            [
              "key"
            ]
          ],
          "aliasName": "relationship_types",
          "selected": false
        },
        "stream": {
          "name": "relationship_types",
          "jsonSchema": {
            "type": "object",
            "properties": {
              "key": {
                "type": "string"
              },
              "source": {
                "type": "string"
              },
              "target": {
                "type": "string"
              },
              "created_at": {
                "type": "string"
              },
              "updated_at": {
                "type": "string"
              }
            }
          },
          "supportedSyncModes": [
            "full_refresh"
          ],
          "sourceDefinedCursor": null,
          "defaultCursorField": [],
          "sourceDefinedPrimaryKey": [
            [
              "key"
            ]
          ],
          "namespace": null
        }
      }
    ]
  },
  "prefix": "",
  "namespaceDefinition": "source",
  "namespaceFormat": "${SOURCE_NAMESPACE}",
  "schedule": null,
  "operations": [
    {
      "workspaceId": "11de4306-c629-4c4b-abde-698e67ce9d77",
      "operationId": "8432ed45-5dbb-43fc-9a26-93c0fd265655",
      "name": "Normalization",
      "operatorConfiguration": {
        "operatorType": "normalization",
        "normalization": {
          "option": "basic"
        },
        "dbt": null
      }
    }
  ],
  "connectionId": "28221aa0-10d4-4ea9-977a-5c4e2652e7aa",
  "status": "active",
  "withRefreshedCatalog": true
}


Ran sync thrice without making any changes in the object record data. Updated at date is same in every sync, therefore it should not replicate data again in 2nd and 3rd sync but it still does that.

Syncs

Data at destination

Hello Mahak thanks for reporting this. I was able to reproduce the issue.

Using Incremental Append I found a duplicate record for Zendesk Sunshine.

Created the issue https://github.com/airbytehq/airbyte/issues/16460 to solve the issue.

Hi @marcosmarxm,

Thank you for looking into this issue, Would be great if i get notified when the issue is resolved,

I am still able to reproduce this issue in 0.40.04 airbyte version

You’ll be notified as soon the issue is solved.