sparkdq-community · flitzpiepe93 · May 12, 2025 · May 11, 2025 · May 11, 2025 · May 11, 2025
diff --git a/docs/source/_static/integration_patterns.png b/docs/source/_static/integration_patterns.png
diff --git a/docs/source/built_in_checks/aggregate.rst b/docs/source/built_in_checks/aggregate.rst
@@ -0,0 +1,25 @@
+Aggregate Checks
+================
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Built-in Checks
+   :hidden:
+
+   checks/schema/column_presence_check
+   checks/count/count_between_check
+   checks/count/count_exact_check
+   checks/count/count_min_check
+   checks/count/count_max_check
+   checks/schema/schema_check
+
+.. csv-table::
+    :header: "Check", "Description"
+    :widths: 20, 80
+
+    ":ref:`count-min-check` ", "Ensures that the DataFrame contains at least a defined minimum number of rows."
+    ":ref:`count-max-check` ", "Ensures that the DataFrame does not exceed a defined maximum number of rows."
+    ":ref:`count-between-check` ", "Ensures that the number of rows in the dataset falls within a defined inclusive range."
+    ":ref:`count-exact-check` ", "Ensures that the dataset contains exactly the specified number of rows."
+    ":ref:`column-presence-check` ", "Verifies the existence of required columns in the DataFrame, independent of their data types."
+    ":ref:`schema-check` ", "Ensures that a DataFrame matches an expected schema by verifying column names and data types, with optional strict enforcement against unexpected columns."
diff --git a/...checks/columns_comparison/column_less.rst → ...checks/columns_comparison/column_less.rst b/...checks/columns_comparison/column_less.rst → ...checks/columns_comparison/column_less.rst
diff --git a/...ks/contained_in/is_contained_in_check.rst → ...ks/contained_in/is_contained_in_check.rst b/...ks/contained_in/is_contained_in_check.rst → ...ks/contained_in/is_contained_in_check.rst
@@ -33,7 +33,7 @@ Declarative Configuration
 
     - check: is-contained-in-check
       check-id: validate_status_and_country
-      allowed_values:
+      allowed-values:
         status:
           - ACTIVE
           - INACTIVE

diff --git a/...ontained_in/is_not_contained_in_check.rst → ...ontained_in/is_not_contained_in_check.rst b/...ontained_in/is_not_contained_in_check.rst → ...ontained_in/is_not_contained_in_check.rst
@@ -33,7 +33,7 @@ Declarative Configuration
 
     - check: is-not-contained-in-check
       check-id: block_test_status_and_invalid_countries
-      forbidden_values:
+      forbidden-values:
         status:
           - TEST
           - DUMMY

diff --git a/...rted/checks/count/count_between_check.rst → ...ecks/checks/count/count_between_check.rst b/...rted/checks/count/count_between_check.rst → ...ecks/checks/count/count_between_check.rst
@@ -31,8 +31,8 @@ Declarative Configuration
 
     - check: row-count-between-check
       check-id: expected_daily_batch_size
-      min_count: 1000
-      max_count: 5000
+      min-count: 1000
+      max-count: 5000
       severity: error
 
 Typical Use Cases

diff --git a/...tarted/checks/count/count_exact_check.rst → ...checks/checks/count/count_exact_check.rst b/...tarted/checks/count/count_exact_check.rst → ...checks/checks/count/count_exact_check.rst
@@ -30,7 +30,7 @@ Declarative Configuration
 
     - check: row-count-exact-check
       check-id: validate_snapshot_size
-      expected_count: 500
+      expected-count: 500
       severity: error
 
 Typical Use Cases

diff --git a/..._started/checks/count/count_max_check.rst → ...n_checks/checks/count/count_max_check.rst b/..._started/checks/count/count_max_check.rst → ...n_checks/checks/count/count_max_check.rst
@@ -30,7 +30,7 @@ Declarative Configuration
 
     - check: row-count-max-check
       check-id: prevent_oversize_batch
-      max_count: 100000
+      max-count: 100000
       severity: error
 
 

diff --git a/..._started/checks/count/count_min_check.rst → ...n_checks/checks/count/count_min_check.rst b/..._started/checks/count/count_min_check.rst → ...n_checks/checks/count/count_min_check.rst
@@ -30,7 +30,7 @@ Declarative Configuration
 
     - check: row-count-min-check
       check-id: minimum_required_records
-      min_count: 10000
+      min-count: 10000
       severity: warning
 
 Typical Use Cases

diff --git a/...tarted/checks/date/date_between_check.rst → ...checks/checks/date/date_between_check.rst b/...tarted/checks/date/date_between_check.rst → ...checks/checks/date/date_between_check.rst
@@ -42,8 +42,8 @@ Declarative Configuration
       check-id: allowed_record_date_range
       columns:
         - record_date
-      min_value: "2020-01-01"
-      max_value: "2023-12-31"
+      min-value: "2020-01-01"
+      max-value: "2023-12-31"
       inclusive: [true, true]
       severity: critical
 

diff --git a/...ng_started/checks/date/date_max_check.rst → ..._in_checks/checks/date/date_max_check.rst b/...ng_started/checks/date/date_max_check.rst → ..._in_checks/checks/date/date_max_check.rst
@@ -39,7 +39,7 @@ Declarative Configuration
       check-id: maximum_allowed_record_date
       columns:
         - record_date
-      max_value: "2023-12-31"
+      max-value: "2023-12-31"
       inclusive: true
       severity: critical
 

diff --git a/...ng_started/checks/date/date_min_check.rst → ..._in_checks/checks/date/date_min_check.rst b/...ng_started/checks/date/date_min_check.rst → ..._in_checks/checks/date/date_min_check.rst
@@ -39,7 +39,7 @@ Declarative Configuration
       check-id: minimum_allowed_record_date
       columns:
         - record_date
-      min_value: "2020-01-01"
+      min-value: "2020-01-01"
       inclusive: true
       severity: critical
 

diff --git a/...hecks/null/exactly_one_not_null_check.rst → ...hecks/null/exactly_one_not_null_check.rst b/...hecks/null/exactly_one_not_null_check.rst → ...hecks/null/exactly_one_not_null_check.rst
diff --git a/...ng_started/checks/null/not_null_check.rst → ..._in_checks/checks/null/not_null_check.rst b/...ng_started/checks/null/not_null_check.rst → ..._in_checks/checks/null/not_null_check.rst
diff --git a/...etting_started/checks/null/null_check.rst → ...uilt_in_checks/checks/null/null_check.rst b/...etting_started/checks/null/null_check.rst → ...uilt_in_checks/checks/null/null_check.rst
diff --git a/.../checks/numeric/numeric_between_check.rst → .../checks/numeric/numeric_between_check.rst b/.../checks/numeric/numeric_between_check.rst → .../checks/numeric/numeric_between_check.rst
@@ -42,8 +42,8 @@ Declarative Configuration
       check-id: allowed_discount_range
       columns:
         - discount
-      min_value: 0.0
-      max_value: 100.0
+      min-value: 0.0
+      max-value: 100.0
       inclusive: [true, true]
       severity: critical
 

diff --git a/...rted/checks/numeric/numeric_max_check.rst → ...ecks/checks/numeric/numeric_max_check.rst b/...rted/checks/numeric/numeric_max_check.rst → ...ecks/checks/numeric/numeric_max_check.rst
@@ -39,7 +39,7 @@ Declarative Configuration
       check-id: maximum_allowed_discount
       columns:
         - discount
-      max_value: 100.0
+      max-value: 100.0
       inclusive: true
       severity: critical
 

diff --git a/...rted/checks/numeric/numeric_min_check.rst → ...ecks/checks/numeric/numeric_min_check.rst b/...rted/checks/numeric/numeric_min_check.rst → ...ecks/checks/numeric/numeric_min_check.rst
@@ -40,7 +40,7 @@ Declarative Configuration
       columns:
         - price
         - discount
-      min_value: 0.0
+      min-value: 0.0
       inclusive: true
       severity: critical
 

diff --git a/...d/checks/schema/column_presence_check.rst → ...s/checks/schema/column_presence_check.rst b/...d/checks/schema/column_presence_check.rst → ...s/checks/schema/column_presence_check.rst
@@ -30,7 +30,7 @@ Declarative Configuration
 
     - check: column-presence-check
       check-id: enforce_required_columns
-      required_columns:
+      required-columns:
         - id
         - event_timestamp
         - status

diff --git a/...ng_started/checks/schema/schema_check.rst → ..._in_checks/checks/schema/schema_check.rst b/...ng_started/checks/schema/schema_check.rst → ..._in_checks/checks/schema/schema_check.rst
@@ -50,7 +50,7 @@ Declarative Configuration
 
     - check: schema-check
       check-id: enforce_schema_contract
-      expected_schema:
+      expected-schema:
         id: int
         name: string
         amount: decimal(10,2)

diff --git a/...rted/checks/strings/regex-match-check.rst → ...ecks/checks/strings/regex_match_check.rst b/...rted/checks/strings/regex-match-check.rst → ...ecks/checks/strings/regex_match_check.rst
diff --git a/.../checks/strings/string_between_length.rst → .../checks/strings/string_between_length.rst b/.../checks/strings/string_between_length.rst → .../checks/strings/string_between_length.rst
diff --git a/...rted/checks/strings/string_max_length.rst → ...ecks/checks/strings/string_max_length.rst b/...rted/checks/strings/string_max_length.rst → ...ecks/checks/strings/string_max_length.rst
diff --git a/...rted/checks/strings/string_min_length.rst → ...ecks/checks/strings/string_min_length.rst b/...rted/checks/strings/string_min_length.rst → ...ecks/checks/strings/string_min_length.rst
diff --git a/...cks/timestamp/timestamp_between_check.rst → ...cks/timestamp/timestamp_between_check.rst b/...cks/timestamp/timestamp_between_check.rst → ...cks/timestamp/timestamp_between_check.rst
@@ -42,8 +42,8 @@ Declarative Configuration
       check-id: allowed_event_time_range
       columns:
         - event_time
-      min_value: "2020-01-01 00:00:00"
-      max_value: "2023-12-31 23:59:59"
+      min-value: "2020-01-01 00:00:00"
+      max-value: "2023-12-31 23:59:59"
       inclusive: [true, true]
       severity: critical
 

diff --git a/.../checks/timestamp/timestamp_max_check.rst → .../checks/timestamp/timestamp_max_check.rst b/.../checks/timestamp/timestamp_max_check.rst → .../checks/timestamp/timestamp_max_check.rst
@@ -39,7 +39,7 @@ Declarative Configuration
       check-id: maximum_allowed_event_time
       columns:
         - event_time
-      max_value: "2023-12-31 23:59:59"
+      max-value: "2023-12-31 23:59:59"
       inclusive: true
       severity: critical
 

diff --git a/.../checks/timestamp/timestamp_min_check.rst → .../checks/timestamp/timestamp_min_check.rst b/.../checks/timestamp/timestamp_min_check.rst → .../checks/timestamp/timestamp_min_check.rst
@@ -40,7 +40,7 @@ Declarative Configuration
       check-id: minimum_allowed_event_time
       columns:
         - event_time
-      min_value: "2020-01-01 00:00:00"
+      min-value: "2020-01-01 00:00:00"
       inclusive: true
       severity: critical
 

diff --git a/...ource/getting_started/built_in_checks.rst → docs/source/built_in_checks/row_level.rst b/...ource/getting_started/built_in_checks.rst → docs/source/built_in_checks/row_level.rst
@@ -1,129 +1,50 @@
-Built-In Checks
-===============
-
-SparkDQ includes various built-in checks for validating the integrity and quality of your data.
-The following table lists all available checks with their identifiers and a brief
-description — click on a check name to see details and examples.
-
-All configuration examples are shown in YAML format for better readability.
-You can still define checks via JSON, or external sources — as long as the
-configuration is provided as a Python dictionary at runtime.
+Row-Level Checks
+================
 
 .. toctree::
    :maxdepth: 1
    :caption: Built-in Checks
    :hidden:
 
    checks/columns_comparison/column_less
-   checks/schema/column_presence_check
-   checks/count/count_between_check
-   checks/count/count_exact_check
-   checks/count/count_min_check
-   checks/count/count_max_check
-
    checks/date/date_between_check
-   checks/date/date_min_check
    checks/date/date_max_check
-
+   checks/date/date_min_check
    checks/null/exactly_one_not_null_check
-
    checks/contained_in/is_contained_in_check
    checks/contained_in/is_not_contained_in_check
-
    checks/null/not_null_check
    checks/null/null_check
-   checks/numeric/numeric_min_check
-   checks/numeric/numeric_max_check
    checks/numeric/numeric_between_check
-
+   checks/numeric/numeric_max_check
+   checks/numeric/numeric_min_check
    checks/strings/regex_match_check
-
-   checks/schema/schema_check
    checks/strings/string_between_length
    checks/strings/string_max_length
    checks/strings/string_min_length
-
-   checks/timestamp/timestamp_min_check
-   checks/timestamp/timestamp_max_check
    checks/timestamp/timestamp_between_check
-
-Comparison Checks
------------------
-
-.. csv-table::
-    :header: "Check", "Description"
-    :widths: 20, 80
-
-    ":ref:`column_less_than_check`", "Validates that values in one column are strictly less than (or less than or equal to) the values in another column."
-
-Contained-In Checks
-----------------------
+   checks/timestamp/timestamp_max_check
+   checks/timestamp/timestamp_min_check
 
 .. csv-table::
     :header: "Check", "Description"
     :widths: 20, 80
 
+    ":ref:`date-between-check` ", "Ensures that date column values are within a defined range."
+    ":ref:`date-max-check` ", "Ensures that date column values are less than a defined maximum date."
+    ":ref:`date-min-check` ", "Ensures that date column values are greater than a defined minimum date."
+    ":ref:`exactly_one_not_null_check` ", "Validates that exactly one of the specified columns is non-null per row."
     ":ref:`is-contained-in-check` ", "Ensures that column values are contained within a predefined set of allowed values."
     ":ref:`is-not-contained-in-check` ", "Ensures that column values are not contained within a set of forbidden values."
-
-Count Checks
-------------
-
-.. csv-table::
-    :header: "Check", "Description"
-    :widths: 20, 80
-
-    ":ref:`count-min-check` ", "Ensures that the DataFrame contains at least a defined minimum number of rows."
-    ":ref:`count-max-check` ", "Ensures that the DataFrame does not exceed a defined maximum number of rows."
-    ":ref:`count-between-check` ", "Ensures that the number of rows in the dataset falls within a defined inclusive range."
-    ":ref:`count-exact-check` ", "Ensures that the dataset contains exactly the specified number of rows."
-
-Null Checks
------------
-
-.. csv-table::
-    :header: "Check", "Description"
-    :widths: 20, 80
-
-    ":ref:`null_check` ", "Verifies whether a given column contains any null values."
     ":ref:`not_null_check` ", "Checks whether the specified column contains at least one non-null value."
-    ":ref:`exactly_one_not_null_check` ", "Validates that exactly one of the specified columns is non-null per row."
-
-Range Checks
-------------
-
-.. csv-table::
-    :header: "Check", "Description"
-    :widths: 20, 80
-
-    ":ref:`numeric-min-check` ", "Ensures that numeric column values are greater than a defined minimum."
-    ":ref:`numeric-max-check` ", "Ensures that numeric column values are less than a defined maximum."
+    ":ref:`null_check` ", "Verifies whether a given column contains any null values."
     ":ref:`numeric-between-check` ", "Ensures that numeric column values are within a defined inclusive range."
-    ":ref:`date-min-check` ", "Ensures that date column values are greater than a defined minimum date."
-    ":ref:`date-max-check` ", "Ensures that date column values are less than a defined maximum date."
-    ":ref:`date-between-check` ", "Ensures that date column values are within a defined range."
-    ":ref:`timestamp-min-check` ", "Ensures that timestamp column values are greater than a defined timestamp."
-    ":ref:`timestamp-max-check` ", "Ensures that timestamp column values are less than a defined maximum timestamp."
+    ":ref:`numeric-max-check` ", "Ensures that numeric column values are less than a defined maximum."
+    ":ref:`numeric-min-check` ", "Ensures that numeric column values are greater than a defined minimum."
     ":ref:`timestamp-between-check` ", "Ensures that timestamp column values are within a defined inclusive range between a minimum and maximum timestamp."
-
-Schema Checks
--------------
-
-.. csv-table::
-    :header: "Check", "Description"
-    :widths: 20, 80
-
-    ":ref:`column-presence-check` ", "Verifies the existence of required columns in the DataFrame, independent of their data types."
-    ":ref:`schema-check` ", "Ensures that a DataFrame matches an expected schema by verifying column names and data types, with optional strict enforcement against unexpected columns."
-
-String Checks
--------------
-
-.. csv-table::
-    :header: "Check", "Description"
-    :widths: 20, 80
-
+    ":ref:`timestamp-max-check` ", "Ensures that timestamp column values are less than a defined maximum timestamp."
+    ":ref:`timestamp-min-check` ", "Ensures that timestamp column values are greater than a defined timestamp."
     ":ref:`regex_match_check`", "Validates that string values match a given regular expression pattern."
-    ":ref:`string_min_length_check`", "Validates that string values in a column meet a minimum length requirement."
-    ":ref:`string_max_length_check`", "Ensures that string values do not exceed a maximum length."
     ":ref:`string_length_between_check`", "Checks that string lengths fall within a specified range."
+    ":ref:`string_max_length_check`", "Ensures that string values do not exceed a maximum length."
+    ":ref:`string_min_length_check`", "Validates that string values in a column meet a minimum length requirement."
diff --git a/docs/source/custom_checks/implementation/aggregate.rst b/docs/source/custom_checks/implementation/aggregate.rst
@@ -75,7 +75,7 @@ Minimal Example
    @register_check_config(check_name="my-custom-count-check")
    class RowCountMinCheckConfig(BaseAggregateCheckConfig):
        check_class = RowCountMinCheck
-       min_count: int = Field(..., description="Minimum number of rows expected")
+       min_count: int = Field(..., description="Minimum number of rows expected", alias="min-count")
 
        @model_validator(mode="after")
        def validate_min(self) -> "RowCountMinCheckConfig":

diff --git a/docs/source/custom_checks/plugin_architecture.rst b/docs/source/custom_checks/plugin_architecture.rst
@@ -33,8 +33,8 @@ Workflow
    .. code-block:: python
 
       config = [
-          {"check": "null-check", "check_id": "c1", "column": "age"},
-          {"check": "positive-value", "check_id": "c2", "column": "salary"}
+          {"check": "null-check", "check-id": "c1", "column": "age"},
+          {"check": "positive-value", "check-id": "c2", "column": "salary"}
       ]
 
 2. For each entry, it uses the ``check`` to look up the matching `CheckConfig` class via the central **registry**.

diff --git a/docs/source/getting_started/applying_validation.rst b/docs/source/getting_started/applying_validation.rst
@@ -9,6 +9,11 @@ different destinations.
 
 Here are two practical approaches that you can implement with just a few lines of code.
 
+.. image:: /_static/integration_patterns.png
+   :alt: Integration patterns diagram
+   :align: center
+   :width: 100%
+
 Fail-Fast Validation
 --------------------
 

diff --git a/docs/source/getting_started/defining_checks.rst b/docs/source/getting_started/defining_checks.rst
@@ -121,8 +121,8 @@ definitions via dictionaries — for example loaded from YAML or JSON files.
 
     - check: row-count-between-check
       check-id: my-count-check
-      min_count: 100
-      max_count: 5000
+      min-count: 100
+      max-count: 5000
 
 To load the configuration into SparkDQ, use the following code:
 

diff --git a/docs/source/getting_started/validation_dataframes.rst b/docs/source/getting_started/validation_dataframes.rst
@@ -21,7 +21,7 @@ Here’s how it works:
 .. code-block:: python
 
     from sparkdq.checks import NullCheckConfig
-    from sparkdq.core import Severity
+    from sparkdq.engine import BatchDQEngine
     from sparkdq.management import CheckSet
 
     check_set = CheckSet()
@@ -81,8 +81,6 @@ sparkdq automatically annotates your DataFrame with additional columns:
 
 * ``_dq_errors``: Array of structured errors for each failed check (name, check-id, severity)
 
-* ``_dq_aggregate_errors``: Optional column for dataset-wide violations
-
 * ``_dq_validation_ts``: Timestamp marking when the validation run was executed
 
 This enriched metadata allows you to: