From d71b0aaa168e38927067536c6e5e0368ece70c05 Mon Sep 17 00:00:00 2001 From: Anna Saiapina Date: Thu, 21 Nov 2024 09:40:07 +0000 Subject: [PATCH] - Add assertion for freshness of data checking - Add filters on csc stg table --- definitions/assertions/csc_rev_data_freshness.sqlx | 9 +++++++++ definitions/staging/stg_csc_agent.sqlx | 3 +++ 2 files changed, 12 insertions(+) create mode 100644 definitions/assertions/csc_rev_data_freshness.sqlx diff --git a/definitions/assertions/csc_rev_data_freshness.sqlx b/definitions/assertions/csc_rev_data_freshness.sqlx new file mode 100644 index 0000000..0359c6e --- /dev/null +++ b/definitions/assertions/csc_rev_data_freshness.sqlx @@ -0,0 +1,9 @@ +config { + type: "assertion", + tags: ["csc_rev_changes"] } + +select * from( +select max(CHANGE_DATE) MAX_CHANGE_DATE +from ${ref("pphe_five9_raw","csc_rev_changes_*")} +where _TABLE_SUFFIX = FORMAT_DATE('%Y%m%d',current_date("Europe/Amsterdam"))) +where TIMESTAMP_DIFF(safe_cast(MAX_CHANGE_DATE as timestamp), TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 DAY), hour) < 0 \ No newline at end of file diff --git a/definitions/staging/stg_csc_agent.sqlx b/definitions/staging/stg_csc_agent.sqlx index 4ba695b..4abfc90 100644 --- a/definitions/staging/stg_csc_agent.sqlx +++ b/definitions/staging/stg_csc_agent.sqlx @@ -6,6 +6,7 @@ config { uniqueKey: ["UPDATE_USER", "CONFIRMATION_NO", "CHANGE_DATE", "UPDATE_TYPE"], nonNull: ["UPDATE_USER", "CONFIRMATION_NO", "CHANGE_DATE", "UPDATE_TYPE"] }, + dependencies: ["csc_rev_data_freshness"], description: "All history data for csc revenue changes" } @@ -47,3 +48,5 @@ on cr.RESV_STATUS=ms.RESV_STATUS left join ${ref("pphe_five9_stg","stg_MAPPING_UPDATE_TYPE")} mu on cr.UPDATE_TYPE=mu.UPDATE_TYPE where row_number = 1 +and cr.UPDATE_USER is not null +and ca.AGENT_ID is not null