parametrize pipeline

Author: PowerAtlas

steps/01_setup_snowflake.sql

@@ -13,14 +13,14 @@ CREATE OR ALTER DATABASE QUICKSTART_COMMON;
 -- API integration is needed for GitHub integration
 CREATE OR REPLACE API INTEGRATION git_api_integration
   API_PROVIDER = git_https_api
-  API_ALLOWED_PREFIXES = ('https://github.com/<insert GitHub username>') -- INSERT YOUR GITHUB USERNAME HERE
+  API_ALLOWED_PREFIXES = ('https://github.com/PowerAtlas') -- INSERT YOUR GITHUB USERNAME HERE
   ENABLED = TRUE;
 
 
 -- Git repository object is similar to external stage
 CREATE OR REPLACE GIT REPOSITORY quickstart_common.public.quickstart_repo
   API_INTEGRATION = git_api_integration
-  ORIGIN = '<insert URL of forked GitHub repo>'; -- INSERT URL OF FORKED REPO HERE
+  ORIGIN = 'https://github.com/PowerAtlas/sfguide-getting-started-with-snowflake-devops.git'; -- INSERT URL OF FORKED REPO HERE
 
 
 CREATE OR ALTER DATABASE QUICKSTART_PROD;

steps/03_harmonize_data.py

@@ -245,3 +245,32 @@ def main(df):
 )
 for view in pipeline:
     silver_schema.views.create(view, mode=CreateMode.or_replace)
+
+View(
+    name="attractions",
+    columns=[
+        ViewColumn(name="geo_id"),
+        ViewColumn(name="geo_name"),
+        ViewColumn(name="aquarium_cnt"),
+        ViewColumn(name="zoo_cnt"),
+        ViewColumn(name="korean_restaurant_cnt"),
+    ],
+    query="""
+    select
+        city.geo_id,
+        city.geo_name,
+        count(case when category_main = 'Aquarium' THEN 1 END) aquarium_cnt,
+        count(case when category_main = 'Zoo' THEN 1 END) zoo_cnt,
+        count(case when category_main = 'Korean Restaurant' THEN 1 END) korean_restaurant_cnt,
+    from us_addresses__poi.cybersyn.point_of_interest_index poi
+    join us_addresses__poi.cybersyn.point_of_interest_addresses_relationships poi_add 
+        on poi_add.poi_id = poi.poi_id
+    join us_addresses__poi.cybersyn.us_addresses address 
+        on address.address_id = poi_add.address_id
+    join major_us_cities city on city.geo_id = address.id_city
+    where true
+        and category_main in ('Aquarium', 'Zoo', 'Korean Restaurant')
+        and id_country = 'country/USA'
+    group by city.geo_id, city.geo_name
+    """,
+),

steps/04_orchestrate_jobs.sql

@@ -13,6 +13,9 @@ create or alter table vacation_spots (
   , avg_cloud_cover_pct float
   , precipitation_probability_pct float
   -- STEP 5: INSERT CHANGES HERE
+  , aquarium_cnt int
+  , zoo_cnt int
+  , korean_restaurant_cnt int
 ) data_retention_time_in_days = 1;
 
 
@@ -26,6 +29,7 @@ create or alter task vacation_spots_update
     from silver.flights_from_home flight
     join silver.weather_joined_with_major_cities city on city.geo_name = flight.arrival_city
     -- STEP 5: INSERT CHANGES HERE
+    join silver.attractions att on att.geo_name = city.geo_name
   ) as harmonized_vacation_spots ON vacation_spots.city = harmonized_vacation_spots.arrival_city and vacation_spots.airport = harmonized_vacation_spots.arrival_airport
   WHEN MATCHED THEN
     UPDATE SET
@@ -36,6 +40,9 @@ create or alter task vacation_spots_update
       , vacation_spots.avg_cloud_cover_pct = harmonized_vacation_spots.avg_cloud_cover_pct
       , vacation_spots.precipitation_probability_pct = harmonized_vacation_spots.precipitation_probability_pct
       -- STEP 5: INSERT CHANGES HERE
+      , vacation_spots.aquarium_cnt = harmonized_vacation_spots.aquarium_cnt
+      , vacation_spots.zoo_cnt = harmonized_vacation_spots.zoo_cnt
+      , vacation_spots.korean_restaurant_cnt = harmonized_vacation_spots.korean_restaurant_cnt
   WHEN NOT MATCHED THEN 
     INSERT VALUES (
         harmonized_vacation_spots.arrival_city
@@ -47,6 +54,9 @@ create or alter task vacation_spots_update
       , harmonized_vacation_spots.avg_cloud_cover_pct
       , harmonized_vacation_spots.precipitation_probability_pct
       -- STEP 5: INSERT CHANGES HERE
+      , harmonized_vacation_spots.aquarium_cnt
+      , harmonized_vacation_spots.zoo_cnt
+      , harmonized_vacation_spots.korean_restaurant_cnt
     );
 
 
@@ -64,13 +74,15 @@ create or alter task email_notification
           and punctual_pct >= 50
           and avg_temperature_air_f >= 70
           -- STEP 5: INSERT CHANGES HERE
+          and korean_restaurant_cnt > 0
+          and (zoo_cnt > 0 or aquarium_cnt > 0)
         limit 10);
 
 
       if (:options = '[]') then
         CALL SYSTEM$SEND_EMAIL(
             'email_integration',
-            '<insert your email here>', -- INSERT YOUR EMAIL HERE
+            '[email protected]', -- INSERT YOUR EMAIL HERE
             'New data successfully processed: No suitable vacation spots found.',
             'The query did not return any results. Consider adjusting your filters.');
       end if;
@@ -83,14 +95,14 @@ create or alter task email_notification
 
       CALL SYSTEM$SEND_EMAIL(
         'email_integration',
-        '<insert your email here>', -- INSERT YOUR EMAIL HERE
+        '[email protected]', -- INSERT YOUR EMAIL HERE
         'New data successfully processed: The perfect place for your summer vacation has been found.',
         :response);
     exception
         when EXPRESSION_ERROR then
             CALL SYSTEM$SEND_EMAIL(
             'email_integration',
-            '<insert your email here>', -- INSERT YOUR EMAIL HERE
+            '[email protected]', -- INSERT YOUR EMAIL HERE
             'New data successfully processed: Cortex LLM function inaccessible.',
             'It appears that the Cortex LLM functions are not available in your region');
     end;
@@ -105,7 +117,7 @@ alter task email_notification resume;
 execute task vacation_spots_update;
 
 
-/*
+
 -- SQL commands to monitor the progress of tasks
 
 -- Get a list of tasks
@@ -127,4 +139,4 @@ SELECT
 FROM TABLE(INFORMATION_SCHEMA.TASK_HISTORY())
 WHERE STATE = 'SCHEDULED'
 ORDER BY COMPLETED_TIME DESC;
-*/
+