[{"id":"74345","title":"Introduction","description":"

In this module, we introduce the course and agenda

","steps":[{"id":"516206","prompt":null,"isOptional":true,"activities":[{"id":"509028","href":"/paths/16/course_templates/53/video/509028","isLocked":false,"duration":55000,"title":"Course Introduction","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false}],"expanded":false},{"id":"74346","title":"Introduction to Building Batch Data Pipelines","description":"

This module reviews different methods of data loading: EL, ELT and ETL and when to use what

","steps":[{"id":"516207","prompt":null,"isOptional":true,"activities":[{"id":"509029","href":"/paths/16/course_templates/53/video/509029","isLocked":false,"duration":69000,"title":"Module introduction","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516208","prompt":null,"isOptional":true,"activities":[{"id":"509030","href":"/paths/16/course_templates/53/video/509030","isLocked":false,"duration":220000,"title":"EL, ELT, ETL","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516209","prompt":null,"isOptional":true,"activities":[{"id":"509031","href":"/paths/16/course_templates/53/video/509031","isLocked":false,"duration":168000,"title":"Quality considerations","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516210","prompt":null,"isOptional":true,"activities":[{"id":"509032","href":"/paths/16/course_templates/53/video/509032","isLocked":false,"duration":181000,"title":"How to carry out operations in BigQuery","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516211","prompt":null,"isOptional":true,"activities":[{"id":"509033","href":"/paths/16/course_templates/53/video/509033","isLocked":false,"duration":218000,"title":"Shortcomings","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516212","prompt":null,"isOptional":true,"activities":[{"id":"509034","href":"/paths/16/course_templates/53/video/509034","isLocked":false,"duration":403000,"title":"ETL to solve data quality issues","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516213","prompt":null,"isOptional":false,"activities":[{"id":"509035","href":"/paths/16/course_templates/53/quizzes/509035","isLocked":false,"duration":300000,"title":"Introduction to Building Batch Data Pipelines","description":null,"type":"quiz","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false}],"expanded":false},{"id":"74347","title":"Executing Spark on Dataproc","description":"

This module shows how to run Hadoop on Dataproc, how to leverage Cloud Storage, and how to optimize your Dataproc jobs.

","steps":[{"id":"516214","prompt":null,"isOptional":true,"activities":[{"id":"509036","href":"/paths/16/course_templates/53/video/509036","isLocked":false,"duration":27000,"title":"Module introduction","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516215","prompt":null,"isOptional":true,"activities":[{"id":"509037","href":"/paths/16/course_templates/53/video/509037","isLocked":false,"duration":286000,"title":"The Hadoop ecosystem","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516216","prompt":null,"isOptional":true,"activities":[{"id":"509038","href":"/paths/16/course_templates/53/video/509038","isLocked":false,"duration":677000,"title":"Running Hadoop on Dataproc","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516217","prompt":null,"isOptional":true,"activities":[{"id":"509039","href":"/paths/16/course_templates/53/video/509039","isLocked":false,"duration":380000,"title":"Cloud Storage instead of HDFS","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516218","prompt":null,"isOptional":true,"activities":[{"id":"509040","href":"/paths/16/course_templates/53/video/509040","isLocked":false,"duration":172000,"title":"Optimizing Dataproc","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516219","prompt":null,"isOptional":true,"activities":[{"id":"509041","href":"/paths/16/course_templates/53/video/509041","isLocked":false,"duration":567000,"title":"Optimizing Dataproc storage","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516220","prompt":null,"isOptional":true,"activities":[{"id":"509042","href":"/paths/16/course_templates/53/video/509042","isLocked":false,"duration":294000,"title":"Optimizing Dataproc templates and autoscaling","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516221","prompt":null,"isOptional":true,"activities":[{"id":"509043","href":"/paths/16/course_templates/53/video/509043","isLocked":false,"duration":186000,"title":"Optimizing Dataproc monitoring","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516222","prompt":null,"isOptional":true,"activities":[{"id":"509044","href":"/paths/16/course_templates/53/video/509044","isLocked":false,"duration":28000,"title":"Lab Intro: Running Apache Spark jobs on Dataproc","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516223","prompt":null,"isOptional":false,"activities":[{"id":"509045","href":"/paths/16/course_templates/53/labs/509045","isLocked":false,"duration":5400000,"title":"Running Apache Spark jobs on Cloud Dataproc","description":"This lab focuses on running Apache Spark jobs on Cloud Dataproc.","type":"lab","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516224","prompt":null,"isOptional":true,"activities":[{"id":"509046","href":"/paths/16/course_templates/53/video/509046","isLocked":false,"duration":34000,"title":"Summary","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516225","prompt":null,"isOptional":false,"activities":[{"id":"509047","href":"/paths/16/course_templates/53/quizzes/509047","isLocked":false,"duration":900000,"title":"Executing Spark on Dataproc","description":null,"type":"quiz","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false}],"expanded":false},{"id":"74348","title":"Serverless Data Processing with Dataflow","description":"

This module covers using Dataflow to build your data processing pipelines

","steps":[{"id":"516226","prompt":null,"isOptional":true,"activities":[{"id":"509048","href":"/paths/16/course_templates/53/video/509048","isLocked":false,"duration":53000,"title":"Module introduction","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516227","prompt":null,"isOptional":true,"activities":[{"id":"509049","href":"/paths/16/course_templates/53/video/509049","isLocked":false,"duration":342000,"title":"Introduction to Dataflow","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516228","prompt":null,"isOptional":true,"activities":[{"id":"509050","href":"/paths/16/course_templates/53/video/509050","isLocked":false,"duration":171000,"title":"Why customers value Dataflow","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516229","prompt":null,"isOptional":true,"activities":[{"id":"509051","href":"/paths/16/course_templates/53/video/509051","isLocked":false,"duration":228000,"title":"Building Dataflow pipelines in code","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516230","prompt":null,"isOptional":true,"activities":[{"id":"509052","href":"/paths/16/course_templates/53/video/509052","isLocked":false,"duration":127000,"title":"Key considerations with designing pipelines","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516231","prompt":null,"isOptional":true,"activities":[{"id":"509053","href":"/paths/16/course_templates/53/video/509053","isLocked":false,"duration":188000,"title":"Transforming data with PTransforms","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516232","prompt":null,"isOptional":true,"activities":[{"id":"509054","href":"/paths/16/course_templates/53/video/509054","isLocked":false,"duration":10000,"title":"Lab Intro: Building a Simple Dataflow Pipeline","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516233","prompt":null,"isOptional":true,"activities":[{"id":"509055","href":"/paths/16/course_templates/53/labs/509055","isLocked":false,"duration":5400000,"title":"A Simple Dataflow Pipeline (Python) 2.5","description":"In this lab, you learn how to write a simple Dataflow pipeline and run it both locally and on the cloud.","type":"lab","isComplete":false,"inProgress":false,"score":null,"disabled":false},{"id":"509056","href":"/paths/16/course_templates/53/labs/509056","isLocked":false,"duration":5400000,"title":"Serverless Data Analysis with Dataflow: A Simple Dataflow Pipeline (Java)","description":"In this lab you will open a Dataflow project, use pipeline filtering, and execute the pipeline locally and on the cloud using Java.","type":"lab","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516234","prompt":null,"isOptional":true,"activities":[{"id":"509057","href":"/paths/16/course_templates/53/video/509057","isLocked":false,"duration":335000,"title":"Aggregate with GroupByKey and Combine","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516235","prompt":null,"isOptional":true,"activities":[{"id":"509058","href":"/paths/16/course_templates/53/video/509058","isLocked":false,"duration":21000,"title":"Lab Intro: MapReduce in Beam","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516236","prompt":null,"isOptional":true,"activities":[{"id":"509059","href":"/paths/16/course_templates/53/labs/509059","isLocked":false,"duration":5400000,"title":"MapReduce in Beam (Python) 2.5","description":"In this lab, you learn how to use pipeline options and carry out Map and Reduce operations in Dataflow. ","type":"lab","isComplete":false,"inProgress":false,"score":null,"disabled":false},{"id":"509060","href":"/paths/16/course_templates/53/labs/509060","isLocked":false,"duration":5400000,"title":"Serverless Data Analysis with Beam: MapReduce in Beam (Java)","description":"In this lab you will identify Map and Reduce operations, execute the pipeline, use command line parameters.","type":"lab","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516237","prompt":null,"isOptional":true,"activities":[{"id":"509061","href":"/paths/16/course_templates/53/video/509061","isLocked":false,"duration":275000,"title":"Side inputs and windows of data","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516238","prompt":null,"isOptional":true,"activities":[{"id":"509062","href":"/paths/16/course_templates/53/video/509062","isLocked":false,"duration":17000,"title":"Lab Intro: Serverless Data Analysis with Dataflow: Side Inputs","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516239","prompt":null,"isOptional":true,"activities":[{"id":"509063","href":"/paths/16/course_templates/53/labs/509063","isLocked":false,"duration":5400000,"title":"Serverless Data Analysis with Dataflow: Side Inputs (Python)","description":"In this lab you will try out a BigQuery query, explore the pipeline code, and execute the pipeline using Python.","type":"lab","isComplete":false,"inProgress":false,"score":null,"disabled":false},{"id":"509064","href":"/paths/16/course_templates/53/labs/509064","isLocked":false,"duration":5400000,"title":"Serverless Data Analysis with Dataflow: Side Inputs (Java)","description":"In this lab you will try out a BigQuery query, explore the pipeline code, and execute the pipeline using Java.","type":"lab","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516240","prompt":null,"isOptional":true,"activities":[{"id":"509065","href":"/paths/16/course_templates/53/video/509065","isLocked":false,"duration":210000,"title":"Creating and re-using pipeline templates","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516241","prompt":null,"isOptional":true,"activities":[{"id":"509066","href":"/paths/16/course_templates/53/video/509066","isLocked":false,"duration":126000,"title":"Summary","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516242","prompt":null,"isOptional":false,"activities":[{"id":"509067","href":"/paths/16/course_templates/53/quizzes/509067","isLocked":false,"duration":600000,"title":"Serverless Data Processing with Dataflow","description":null,"type":"quiz","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false}],"expanded":false},{"id":"74349","title":"Manage Data Pipelines with Cloud Data Fusion and Cloud Composer","description":"

This module shows how to manage data pipelines with Cloud Data Fusion and Cloud Composer.

","steps":[{"id":"516243","prompt":null,"isOptional":true,"activities":[{"id":"509068","href":"/paths/16/course_templates/53/video/509068","isLocked":false,"duration":19000,"title":"Module introduction","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516244","prompt":null,"isOptional":true,"activities":[{"id":"509069","href":"/paths/16/course_templates/53/video/509069","isLocked":false,"duration":218000,"title":"Introduction to Cloud Data Fusion","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516245","prompt":null,"isOptional":true,"activities":[{"id":"509070","href":"/paths/16/course_templates/53/video/509070","isLocked":false,"duration":74000,"title":"Components of Cloud Data Fusion","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516246","prompt":null,"isOptional":true,"activities":[{"id":"509071","href":"/paths/16/course_templates/53/video/509071","isLocked":false,"duration":95000,"title":"Cloud Data Fusion UI","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516247","prompt":null,"isOptional":true,"activities":[{"id":"509072","href":"/paths/16/course_templates/53/video/509072","isLocked":false,"duration":291000,"title":"Build a pipeline","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516248","prompt":null,"isOptional":true,"activities":[{"id":"509073","href":"/paths/16/course_templates/53/video/509073","isLocked":false,"duration":108000,"title":"Explore data using wrangler","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516249","prompt":null,"isOptional":true,"activities":[{"id":"509074","href":"/paths/16/course_templates/53/video/509074","isLocked":false,"duration":18000,"title":"Lab Intro: Building and executing a pipeline graph in Cloud Data Fusion","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516250","prompt":null,"isOptional":false,"activities":[{"id":"509075","href":"/paths/16/course_templates/53/labs/509075","isLocked":false,"duration":9000000,"title":"Building and Executing a Pipeline Graph with Data Fusion 2.5","description":"This tutorial shows you how to use the Wrangler and Data Pipeline features in Cloud Data Fusion to clean, transform, and process taxi trip data for further analysis.","type":"lab","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516251","prompt":null,"isOptional":true,"activities":[{"id":"509076","href":"/paths/16/course_templates/53/video/509076","isLocked":false,"duration":64000,"title":"Orchestrate work between Google Cloud services with Cloud Composer","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516252","prompt":null,"isOptional":true,"activities":[{"id":"509077","href":"/paths/16/course_templates/53/video/509077","isLocked":false,"duration":74000,"title":"Apache Airflow environment","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516253","prompt":null,"isOptional":true,"activities":[{"id":"509078","href":"/paths/16/course_templates/53/video/509078","isLocked":false,"duration":272000,"title":"DAGs and Operators","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516254","prompt":null,"isOptional":true,"activities":[{"id":"509079","href":"/paths/16/course_templates/53/video/509079","isLocked":false,"duration":301000,"title":"Workflow scheduling","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516255","prompt":null,"isOptional":true,"activities":[{"id":"509080","href":"/paths/16/course_templates/53/video/509080","isLocked":false,"duration":208000,"title":"Monitoring and Logging","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516256","prompt":null,"isOptional":true,"activities":[{"id":"509081","href":"/paths/16/course_templates/53/video/509081","isLocked":false,"duration":22000,"title":"Lab Intro: An Introduction to Cloud Composer","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516257","prompt":null,"isOptional":false,"activities":[{"id":"509082","href":"/paths/16/course_templates/53/labs/509082","isLocked":false,"duration":5400000,"title":"An Introduction to Cloud Composer 2.5","description":"In this lab, you create a Cloud Composer environment using the GCP Console. You then use the Airflow web interface to run a workflow that verifies a data file, creates and runs an Apache Hadoop wordcount job on a Dataproc cluster, and deletes the cluster.","type":"lab","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false},{"id":"516258","prompt":null,"isOptional":false,"activities":[{"id":"509083","href":"/paths/16/course_templates/53/quizzes/509083","isLocked":false,"duration":300000,"title":"Manage Data Pipelines with Cloud Data Fusion and Cloud Composer","description":null,"type":"quiz","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false}],"expanded":false},{"id":"74350","title":"Course Summary","description":"

Course Summary

","steps":[{"id":"516259","prompt":null,"isOptional":true,"activities":[{"id":"509084","href":"/paths/16/course_templates/53/video/509084","isLocked":false,"duration":197000,"title":"Course Summary","description":null,"type":"video","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false}],"expanded":false},{"id":"74351","title":"Course Resources","description":"

PDF links to all modules

","steps":[{"id":"516260","prompt":null,"isOptional":true,"activities":[{"id":"509085","href":"/paths/16/course_templates/53/documents/509085","isLocked":false,"duration":1800000,"title":"Building Batch Data Pipelines on Google Cloud","description":null,"type":"document","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false}],"expanded":false},{"id":"next-steps","title":"Your Next Steps","description":null,"steps":[{"id":"badge-step","prompt":null,"isOptional":true,"activities":[{"id":"badge","href":null,"isLocked":true,"duration":null,"title":"Course Badge","description":null,"type":"badge","isComplete":false,"inProgress":false,"score":null,"disabled":false}],"isComplete":false,"isTestedOut":false,"allActivitiesRequired":false}],"expanded":true}]