- Add assertion for freshness of data checking

- Add filters on csc stg table

definitions/assertions/csc_rev_data_freshness.sqlx

@@ -0,0 +1,9 @@
+config { 
+    type: "assertion",
+    tags: ["csc_rev_changes"] }
+
+select * from(
+select max(CHANGE_DATE) MAX_CHANGE_DATE
+from ${ref("pphe_five9_raw","csc_rev_changes_*")}
+where _TABLE_SUFFIX = FORMAT_DATE('%Y%m%d',current_date("Europe/Amsterdam")))
+where TIMESTAMP_DIFF(safe_cast(MAX_CHANGE_DATE as timestamp), TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 DAY), hour) < 0

definitions/staging/stg_csc_agent.sqlx

@@ -6,6 +6,7 @@ config {
     uniqueKey: ["UPDATE_USER", "CONFIRMATION_NO", "CHANGE_DATE", "UPDATE_TYPE"],
     nonNull: ["UPDATE_USER", "CONFIRMATION_NO", "CHANGE_DATE", "UPDATE_TYPE"]
   },
+  dependencies: ["csc_rev_data_freshness"],
   description: "All history data for csc revenue changes"
 }
 
@@ -47,3 +48,5 @@ on cr.RESV_STATUS=ms.RESV_STATUS
 left join ${ref("pphe_five9_stg","stg_MAPPING_UPDATE_TYPE")} mu
 on cr.UPDATE_TYPE=mu.UPDATE_TYPE
 where row_number = 1
+and cr.UPDATE_USER is not null
+and ca.AGENT_ID is not null