diff --git a/quickstarts/_partials/tabsDBT.mdx b/quickstarts/_partials/tabsDBT.mdx index 475cc40064d..3696183f4ae 100644 --- a/quickstarts/_partials/tabsDBT.mdx +++ b/quickstarts/_partials/tabsDBT.mdx @@ -5,20 +5,20 @@ import TabItem from '@theme/TabItem'; Run in Powershell: ```bash - python -m venv env - .\env\Scripts\activate + uv venv .venv + .venv\Scripts\activate ``` - + ```bash - python3 -m venv env - source env/bin/activate + uv venv .venv + source .venv/bin/activate ``` ```bash - python3 -m venv env - source env/bin/activate + uv venv .venv + source .venv/bin/activate ``` diff --git a/quickstarts/images/dbt-cloud/1-create-project.png b/quickstarts/images/dbt-cloud/1-create-project.png new file mode 100644 index 00000000000..29deb0091fb Binary files /dev/null and b/quickstarts/images/dbt-cloud/1-create-project.png differ diff --git a/quickstarts/images/dbt-cloud/10-deploy-job.png b/quickstarts/images/dbt-cloud/10-deploy-job.png new file mode 100644 index 00000000000..986fac61c91 Binary files /dev/null and b/quickstarts/images/dbt-cloud/10-deploy-job.png differ diff --git a/quickstarts/images/dbt-cloud/11-run-job.png b/quickstarts/images/dbt-cloud/11-run-job.png new file mode 100644 index 00000000000..b98866d029b Binary files /dev/null and b/quickstarts/images/dbt-cloud/11-run-job.png differ diff --git a/quickstarts/images/dbt-cloud/12-final-output.png b/quickstarts/images/dbt-cloud/12-final-output.png new file mode 100644 index 00000000000..afb87543962 Binary files /dev/null and b/quickstarts/images/dbt-cloud/12-final-output.png differ diff --git a/quickstarts/images/dbt-cloud/2.1-add-new-connection.png b/quickstarts/images/dbt-cloud/2.1-add-new-connection.png new file mode 100644 index 00000000000..002c990fff2 Binary files /dev/null and b/quickstarts/images/dbt-cloud/2.1-add-new-connection.png differ diff --git a/quickstarts/images/dbt-cloud/2.2-add-new-connection.png b/quickstarts/images/dbt-cloud/2.2-add-new-connection.png new file mode 100644 index 00000000000..db75fc31e85 Binary files /dev/null and b/quickstarts/images/dbt-cloud/2.2-add-new-connection.png differ diff --git a/quickstarts/images/dbt-cloud/2.3-settings.png b/quickstarts/images/dbt-cloud/2.3-settings.png new file mode 100644 index 00000000000..62b4cce46e6 Binary files /dev/null and b/quickstarts/images/dbt-cloud/2.3-settings.png differ diff --git a/quickstarts/images/dbt-cloud/2.4-connection-and-credentials.png b/quickstarts/images/dbt-cloud/2.4-connection-and-credentials.png new file mode 100644 index 00000000000..c63d45ff622 Binary files /dev/null and b/quickstarts/images/dbt-cloud/2.4-connection-and-credentials.png differ diff --git a/quickstarts/images/dbt-cloud/3-set-up-repo.png b/quickstarts/images/dbt-cloud/3-set-up-repo.png new file mode 100644 index 00000000000..c002347e1b7 Binary files /dev/null and b/quickstarts/images/dbt-cloud/3-set-up-repo.png differ diff --git a/quickstarts/images/dbt-cloud/4-environment.png b/quickstarts/images/dbt-cloud/4-environment.png new file mode 100644 index 00000000000..482e237fe3d Binary files /dev/null and b/quickstarts/images/dbt-cloud/4-environment.png differ diff --git a/quickstarts/images/dbt-cloud/5-env-setting.png b/quickstarts/images/dbt-cloud/5-env-setting.png new file mode 100644 index 00000000000..44be841d7ad Binary files /dev/null and b/quickstarts/images/dbt-cloud/5-env-setting.png differ diff --git a/quickstarts/images/dbt-cloud/6-assign-profile.png b/quickstarts/images/dbt-cloud/6-assign-profile.png new file mode 100644 index 00000000000..c47eb338f86 Binary files /dev/null and b/quickstarts/images/dbt-cloud/6-assign-profile.png differ diff --git a/quickstarts/images/dbt-cloud/7-add-new-con.png b/quickstarts/images/dbt-cloud/7-add-new-con.png new file mode 100644 index 00000000000..6d0eaf8662e Binary files /dev/null and b/quickstarts/images/dbt-cloud/7-add-new-con.png differ diff --git a/quickstarts/images/dbt-cloud/8-test-connection.png b/quickstarts/images/dbt-cloud/8-test-connection.png new file mode 100644 index 00000000000..3bbb88a5205 Binary files /dev/null and b/quickstarts/images/dbt-cloud/8-test-connection.png differ diff --git a/quickstarts/images/dbt-cloud/9-create-job.png b/quickstarts/images/dbt-cloud/9-create-job.png new file mode 100644 index 00000000000..7cff0eed551 Binary files /dev/null and b/quickstarts/images/dbt-cloud/9-create-job.png differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud1.png b/quickstarts/images/dbt-cloud/dbt-cloud1.png deleted file mode 100644 index 0377c9cb074..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud1.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud10.png b/quickstarts/images/dbt-cloud/dbt-cloud10.png deleted file mode 100644 index efcca03ab2f..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud10.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud11.png b/quickstarts/images/dbt-cloud/dbt-cloud11.png deleted file mode 100644 index 4d724e0d33a..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud11.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud12.png b/quickstarts/images/dbt-cloud/dbt-cloud12.png deleted file mode 100644 index 22235263ccd..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud12.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud13.png b/quickstarts/images/dbt-cloud/dbt-cloud13.png deleted file mode 100644 index 9460c759c24..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud13.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud14.png b/quickstarts/images/dbt-cloud/dbt-cloud14.png deleted file mode 100644 index b5f6361cef9..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud14.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud15.png b/quickstarts/images/dbt-cloud/dbt-cloud15.png deleted file mode 100644 index 174bf81688d..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud15.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud2.png b/quickstarts/images/dbt-cloud/dbt-cloud2.png deleted file mode 100644 index 3d254d8b01f..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud2.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud3.png b/quickstarts/images/dbt-cloud/dbt-cloud3.png deleted file mode 100644 index b29da36fe75..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud3.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud4.png b/quickstarts/images/dbt-cloud/dbt-cloud4.png deleted file mode 100644 index 1d455794654..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud4.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud5.png b/quickstarts/images/dbt-cloud/dbt-cloud5.png deleted file mode 100644 index 7026b5fa3b3..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud5.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud6.png b/quickstarts/images/dbt-cloud/dbt-cloud6.png deleted file mode 100644 index 370b84156cc..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud6.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud7.png b/quickstarts/images/dbt-cloud/dbt-cloud7.png deleted file mode 100644 index 19c34aa9d15..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud7.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud8.png b/quickstarts/images/dbt-cloud/dbt-cloud8.png deleted file mode 100644 index 9b53e5aa8eb..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud8.png and /dev/null differ diff --git a/quickstarts/images/dbt-cloud/dbt-cloud9.png b/quickstarts/images/dbt-cloud/dbt-cloud9.png deleted file mode 100644 index d61ee586f6f..00000000000 Binary files a/quickstarts/images/dbt-cloud/dbt-cloud9.png and /dev/null differ diff --git a/quickstarts/manage-data/create-parquet-files-in-object-storage.md b/quickstarts/manage-data/create-parquet-files-in-object-storage.md index ad54c880d70..05c7560dd8e 100644 --- a/quickstarts/manage-data/create-parquet-files-in-object-storage.md +++ b/quickstarts/manage-data/create-parquet-files-in-object-storage.md @@ -1,40 +1,41 @@ --- sidebar_position: 12 id: create-parquet-files-in-object-storage -author: Obed Vega -email: obed.vega@teradata.com -page_last_update: August 2nd, 2022 -description: Teradata Vantage Native Object Storage - read and write from/to object storage, unified SQL interface for Vantage and object storage. -keywords: [data warehouses, compute storage separation, Teradata, vantage, cloud data platform, object storage, business intelligence, enterprise analytics, parquet, create parquet files] +author: Obed Vega, Vidhan Bhonsle +email: developer.relations@teradata.com +page_last_update: July 6th, 2026 +description: Use Teradata Native Object Storage to read from and write to object storage using a unified SQL interface for Teradata and object storage. +keywords: [data warehouses, compute storage separation, Teradata, cloud data platform, object storage, business intelligence, enterprise analytics, parquet, create parquet files] --- import UseCase from '../_partials/use-trials.mdx'; -import CommunityLink from '../_partials/community_link.mdx'; # Create Parquet files in object storage ## Overview -Native Object Storage (NOS) is a Vantage feature that allows you to query data stored in files such as CSV, JSON, and Parquet format datasets. -These datasets are located on external S3-compatible object storage such as AWS S3, Google GCS, Azure Blob or on-prem implementations. -It's useful in scenarios where you want to explore data without building a data pipeline to bring it into Vantage. This tutorial demonstrates how to export data from Vantage to object storage using the Parquet file format. +Native Object Storage (NOS) is a feature of Teradata that allows you to query data stored in external object storage, including CSV, JSON, and Parquet files. + +These datasets can be located in external object storage such as Amazon S3, Google Cloud Storage, Azure Blob Storage, or on-premises object storage. + +NOS is useful when you want to explore external data without first building a data pipeline to load it into Teradata. This tutorial demonstrates how to export data from Teradata to object storage in Parquet format. ## Prerequisites -You need access to a Teradata Vantage instance. NOS is enabled in all Vantage editions from Vantage Express through Developer, DYI to Vantage as a Service starting from version 17.10. +You need access to a Teradata instance. NOS is available in Teradata editions starting from version 17.10. :::info -This tutorial is based on s3 aws object storage. You will need your own s3 bucket with write permissions to complete the tutorial. +This tutorial uses Amazon S3 object storage. To complete the tutorial, you need an S3 bucket with write permissions. ::: ## Create a Parquet file with WRITE_NOS function -`WRITE_NOS` allows you to extract selected or all columns from a database table or from derived results and write to external object storage, such as Amazon S3, Azure Blob storage, Azure Data Lake Storage Gen2, and Google Cloud Storage. This functionality stores data in Parquet format. +`WRITE_NOS` allows you to extract selected or all columns from a database table or query results and write them to external object storage, such as Amazon S3, Azure Blob Storage, Azure Data Lake Storage Gen2, and Google Cloud Storage. This functionality writes data in Parquet format. -You can find more documentation about `WRITE_NOS` functionality in the [NOS documentation](https://docs.teradata.com/r/Teradata-VantageTM-Native-Object-Store-Getting-Started-Guide/June-2022/Writing-Data-to-External-Object-Store). +You can find more documentation about the `WRITE_NOS` functionality in the [NOS documentation](https://docs.teradata.com/r/Enterprise_IntelliFlex_VMware/Native-Object-Store-Getting-Started-Guide/Writing-Data-to-External-Object-Store). -You will need access to a database where you can execute `WRITE_NOS` function. If you don't have such a database, run the following commands: +You need access to a database where you can execute the `WRITE_NOS` function. If you don't have such a database, run the following commands: ``` sql CREATE USER db AS PERM=10e7, PASSWORD=db; @@ -45,11 +46,11 @@ GRANT EXECUTE FUNCTION on TD_SYSFNLIB.WRITE_NOS to db; ``` :::note -If you would like to learn more about setting up users and their privileges, checkout the [NOS documentation](https://docs.teradata.com/r/Teradata-VantageTM-Native-Object-Store-Getting-Started-Guide/June-2022/Setting-Up-Access/Setting-Access-Privileges). +If you would like to learn more about setting up users and their privileges, check out the [NOS documentation](https://docs.teradata.com/r/Teradata-VantageTM-Native-Object-Store-Getting-Started-Guide/June-2022/Setting-Up-Access/Setting-Access-Privileges). ::: -1. Let's first create a table on your Teradata Vantage instance: +1. First create a table on your Teradata instance: ```sql CREATE SET TABLE db.parquet_table ,FALLBACK , @@ -82,17 +83,17 @@ column1 column2 column3 3 22/01/03 3.30 ``` -3. Create the parquet file with `WRITE_NOS`. Don't forget to replace `` with the name of your s3 bucket. Also,replace `` and `` with your access key and secret. +3. Create the Parquet file with `WRITE_NOS`. Replace `` with the name of your S3 bucket. Also, replace `` and `` with your access key and secret. :::note -Check your cloud provider docs how to create credentials to access object storage. For example, for AWS check out [How do I create an AWS access key?](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) +Check your cloud provider documentation to learn how to create credentials to access object storage. For example, for AWS check out [How do I create an AWS access key?](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) ::: ```sql SELECT * FROM WRITE_NOS ( ON ( SELECT * FROM db.parquet_table) USING -LOCATION('/s3/.s3.amazonaws.com/parquet_file_on_NOS.parquet') +LOCATION('/s3/.s3.amazonaws.com/parquet_file_on_NOS/') AUTHORIZATION('{"ACCESS_ID":"", "ACCESS_KEY":""}') STOREDAS('PARQUET') @@ -103,48 +104,64 @@ INCLUDE_HASHBY('TRUE') ) as d; ``` -Now you have created a parquet file in your object storage bucket. Now to easily query your file you need to follow step number 4. +:::note +If you are using temporary AWS credentials, include the session token in the `AUTHORIZATION` string: + +```sql +AUTHORIZATION('{"ACCESS_ID":"", +"ACCESS_KEY":"", +"SESSION_TOKEN":""}') +``` +::: + +Now you have created a Parquet files in your object storage bucket. To query the files, follow step 4. + +4. Create an authorization object. Replace `` and `` with your access key and secret: -4. Create a NOS-backed foreign table. Don't forget to replace `` with the name of your s3 bucket. Also,replace `` and `` with your access key and secret: ```sql -CREATE MULTISET FOREIGN TABLE db.parquet_table_to_read_file_on_NOS -, EXTERNAL SECURITY DEFINER TRUSTED CEPH_AUTH, +CREATE AUTHORIZATION MyAuthObj +USER '' +PASSWORD ''; +``` + +5. Create a NOS-backed foreign table. Replace `` with the name of your S3 bucket: + +```sql +CREATE MULTISET FOREIGN TABLE parquet_table_to_read_file_on_NOS, +EXTERNAL SECURITY MyAuthObj, MAP = TD_MAP1 ( - Location VARCHAR(2048) CHARACTER SET UNICODE CASESPECIFIC - , col1 SMALLINT - , col2 DATE - , col3 DECIMAL(10,2) - + Location VARCHAR(2048) CHARACTER SET UNICODE CASESPECIFIC, + column1 SMALLINT, + column2 DATE, + column3 DECIMAL(10,2) ) USING ( - LOCATION ('/s3/.s3.amazonaws.com/parquet_file_on_NOS.parquet') - AUTHORIZATION('{"ACCESS_ID":"", - "ACCESS_KEY":""}') + LOCATION ('/s3/.s3.amazonaws.com/parquet_file_on_NOS/') STOREDAS ('PARQUET') -)NO PRIMARY INDEX; +) +NO PRIMARY INDEX; ``` -5. Now you are ready to Query your parquet file on NOS, let's try the following query: +6. Query the Parquet files on NOS: + ```sql -SELECT col1, col2, col3 FROM db.parquet_table_to_read_file_on_NOS; +SELECT column1, column2, column3 FROM parquet_table_to_read_file_on_NOS; ``` The data returned from the query should look something like this: ```sql - col1 col2 col3 ------- -------- ------------ - 1 22/01/01 1.10 - 2 22/01/02 2.20 - 3 22/01/03 3.30 +column1 column2 column3 +------- -------- ------------ + 1 22/01/01 1.10 + 2 22/01/02 2.20 + 3 22/01/03 3.30 ``` ## Summary -In this tutorial we have learned how to export data from Vantage to a parquet file on object storage using Native Object Storage (NOS). NOS supports reading and importing data stored in CSV, JSON and Parquet formats. NOS can also export data from Vantage to object storage. +In this tutorial, you learned how to export data from Teradata to object storage in Parquet format using Native Object Storage (NOS). NOS supports reading data stored in CSV, JSON, and Parquet formats. NOS can also export data from Teradata to object storage. ## Further reading -* [Teradata Vantage™ - Writing Data to External Object Store](https://docs.teradata.com/r/Teradata-VantageTM-Native-Object-Store-Getting-Started-Guide/June-2022/Writing-Data-to-External-Object-Store) - - \ No newline at end of file +* [Native Object Storage - Writing Data to External Object Store](https://docs.teradata.com/r/Teradata-VantageTM-Native-Object-Store-Getting-Started-Guide/June-2022/Writing-Data-to-External-Object-Store) \ No newline at end of file diff --git a/quickstarts/manage-data/dbt.md b/quickstarts/manage-data/dbt.md index ee99598a2c5..4095db73107 100644 --- a/quickstarts/manage-data/dbt.md +++ b/quickstarts/manage-data/dbt.md @@ -1,73 +1,70 @@ --- id: dbt sidebar_position: 4.5 -author: Adam Tworkiewicz -email: adam.tworkiewicz@teradata.com -page_last_update: July 12th, 2023 -description: Use dbt (data build tool) with Teradata Vantage. -keywords: [data warehouses, compute storage separation, teradata, vantage, cloud data platform, object storage, business intelligence, enterprise analytics, elt, dbt.] +author: Adam Tworkiewicz, Vidhan Bhonsle +email: developer.relations@teradata.com +page_last_update: July 6th, 2026 +description: Use dbt (data build tool) with Teradata. +keywords: [data warehouses, data transformation, teradata, analytics engineering, business intelligence, enterprise analytics, elt, dbt] --- import TrialDocsNote from '../_partials/teradata_trial.mdx' -import CommunityLink from '../_partials/community_link.mdx' import InstallTabs from '../_partials/tabsDBT.mdx' import ProfileTabs from '../_partials/tabsDBTProfiles.mdx' -# dbt with Teradata Vantage +# dbt with Teradata ## Overview -This tutorial demonstrates how to use dbt (Data Build Tool) with Teradata Vantage. It's based on the original [dbt Jaffle Shop tutorial](https://github.com/dbt-labs/jaffle_shop-dev). A couple of models have been adjusted to the SQL dialect supported by Vantage. +This tutorial demonstrates how to use dbt (data build tool) with Teradata. It's based on the original [dbt Jaffle Shop tutorial](https://github.com/dbt-labs/jaffle_shop-dev). A couple of models have been adjusted to the SQL dialect supported by Teradata. ## Prerequisites -* Access to a Teradata Vantage instance. +* Access to a Teradata instance. -* Python **3.7**, **3.8**, **3.9**, **3.10** or **3.11** installed. +* Python **3.10**, **3.11**, **3.12**, or **3.13** installed. + +* [uv](https://docs.astral.sh/uv/) installed for Python environment and package management. ## Install dbt -1. Clone the tutorial repository and cd into the project directory: - ``` bash +1. Clone the tutorial repository and change into the project directory: + ```bash git clone https://github.com/Teradata/jaffle_shop-dev.git jaffle_shop cd jaffle_shop ``` -2. Create a new python environment to manage dbt and its dependencies. - - Activate the environment: - - +2. Create and activate a new Python virtual environment to manage dbt and its dependencies. -3. Install `dbt-teradata` module and its dependencies. The core dbt module is included as a dependency so you don't have to install it separately: - :::note - **dbt dependencies** + - `dbt-core` module was included as a dependency only up to version 1.7.x of dbt-teradata. Starting from `dbt-teradata` 1.8.0 and above, `dbt-core` will not be installed as a dependency. Therefore, you need to explicitly install `dbt-core` in addition to installing `dbt-teradata`. More information on decoupling dbt adapters from `dbt-core` can be found here: https://github.com/dbt-labs/dbt-core/discussions/9171 - ::: +3. Install `dbt-teradata` and `dbt-core`: + ```bash - pip install dbt-teradata + uv pip install dbt-teradata dbt-core ``` + + ## Configure dbt -Configure dbt to connect to your Vantage database. Create `profiles.yml` file in the location show below. +Configure dbt to connect to your Teradata database. Create a `profiles.yml` file in the location shown below. -Add the following config to `profile.yml` file. Adjust ``, ``, `` to match your Teradata Vantage instance. +Add the following configuration to the `profiles.yml` file. Adjust ``, ``, and `` to match your Teradata instance. :::note **Database setup** The following dbt profile points to a database called `jaffle_shop`. -If the database doesn't exist on your Teradata Vantage instance, it will be created. You can also change `schema` value to point to an existing database in your instance. +If the database doesn't exist on your Teradata Database instance, it will be created. You can also change `schema` value to point to an existing database in your instance. ::: -```bash +```yaml jaffle_shop: outputs: dev: @@ -85,13 +82,13 @@ jaffle_shop: target: dev ``` -Now, that we have the profile file in place, we can validate the setup: +Now that the profile file is in place, validate the setup: ```bash dbt debug ``` -If the debug command returned errors, you likely have an issue with the content of `profiles.yml`. +If the debug command returns errors, you likely have an issue with the content of `profiles.yml`. ## About the Jaffle Shop warehouse @@ -101,7 +98,7 @@ The raw data from the app consists of customers, orders, and payments, with the ![](../images/dbt1.svg) -dbt takes these raw data table and builds the following dimensional model, which is more suitable for analytics tools: +dbt takes these raw data tables and builds the following dimensional model, which is more suitable for analytics tools: ![](../images/dbt2.svg) @@ -109,13 +106,13 @@ dbt takes these raw data table and builds the following dimensional model, which ### Create raw data tables -In real life, we will be getting raw data from platforms like Segment, Stitch, Fivetran or another ETL tool. In our case, we will use dbt's `seed` functionality to create tables from csv files. The csv files are located in `./data` directory. Each csv file will produce one table. dbt will inspect the files and do type inference to decide what data types to use for columns. +In production, raw data is typically ingested from platforms like Segment, Stitch, Fivetran, or another ETL/ELT tool. In this tutorial, we use dbt's `seed` functionality to create tables from CSV files. The CSV files are located in the `./data` directory. Each CSV file produces one table, and dbt infers the column data types from the file contents. ```bash dbt seed ``` -You should now see 3 tables in your `jaffle_shop` database: `raw_customers`, `raw_orders`, `raw_payments`. The tables should be populated with data from the csv files. +You should now see three tables in your `jaffle_shop` database: `raw_customers`, `raw_orders`, and `raw_payments`. The tables should be populated with data from the CSV files. ### Create the dimensional model @@ -124,12 +121,12 @@ Now that we have the raw tables, we can instruct dbt to create the dimensional m dbt run ``` -So what exactly happened here? dbt created additional tables using `CREATE TABLE/VIEW FROM SELECT` SQL. In the first transformation, dbt took raw tables and built denormalized join tables called `customer_orders`, `order_payments`, `customer_payments`. You will find the definitions of these tables in `./marts/core/intermediate`. +So what exactly happened here? dbt created additional tables using `CREATE TABLE/VIEW FROM SELECT` SQL. In the first transformation, dbt took raw tables and built denormalized join tables called `customer_orders`, `order_payments`, `customer_payments`. You will find the definitions of these tables in `./models/marts/core/intermediate`. In the second step, dbt created `dim_customers` and `fct_orders` tables. These are the dimensional model tables that we want to expose to our BI tool. ### Test the data -dbt applied multiple transformations to our data. How can we ensure that the data in the dimensional model is correct? dbt allows us to define and execute tests against the data. The tests are defined in `./marts/core/schema.yml`. The file describes each column in all relationships. Each column can have multiple tests configured under `tests` key. For example, we expect that `fct_orders.order_id` column will contain unique, non-null values. To validate that the data in the produced tables satisfies the test conditions run: +dbt applied multiple transformations to our data. How can we ensure that the data in the dimensional model is correct? dbt allows us to define and execute tests against the data. The tests are defined in `./models/marts/core/schema.yml`. The file describes each column in all relationships. Each column can have multiple tests configured under `tests` key. For example, we expect that `fct_orders.order_id` column will contain unique, non-null values. To validate that the data in the produced tables satisfies the test conditions run: ```bash dbt test @@ -137,13 +134,13 @@ dbt test ### Generate documentation -Our model consists of just a few tables. Imagine a scenario where where we have many more sources of data and a much more complex dimensional model. We could also have an intermediate zone between the raw data and the dimensional model that follows the Data Vault 2.0 principles. Would it not be useful, if we had the inputs, transformations and outputs documented somehow? dbt allows us to generate documentation from its configuration files: +Our model consists of just a few tables. Imagine a scenario where we have many more sources of data and a much more complex dimensional model. We could also have an intermediate zone between the raw data and the dimensional model that follows the Data Vault 2.0 principles. Would it not be useful, if we had the inputs, transformations and outputs documented somehow? dbt allows us to generate documentation from its configuration files: ```bash dbt docs generate ``` -This will produce html files in `./target` directory. +This will produce HTML files in `./target` directory. You can start your own server to browse the documentation. The following command will start a server and open up a browser tab with the docs' landing page: @@ -153,10 +150,8 @@ dbt docs serve ## Summary -This tutorial demonstrated how to use dbt with Teradata Vantage. The sample project takes raw data and produces a dimensional data mart. We used multiple dbt commands to populate tables from csv files (`dbt seed`), create models (`dbt run`), test the data (`dbt test`), and generate and serve model documentation (`dbt docs generate`, `dbt docs serve`). +This tutorial demonstrated how to use dbt with Teradata. The sample project takes raw data and produces a dimensional data mart. We used multiple dbt commands to populate tables from CSV files (`dbt seed`), create models (`dbt run`), test the data (`dbt test`), and generate and serve model documentation (`dbt docs generate`, `dbt docs serve`). ## Further reading * [dbt documentation](https://docs.getdbt.com/docs/) -* [dbt-teradata plugin documentation](https://github.com/Teradata/dbt-teradata) - - +* [dbt-teradata plugin documentation](https://github.com/Teradata/dbt-teradata) \ No newline at end of file diff --git a/quickstarts/manage-data/use-dbt-cloud-with-teradata-vantage.md b/quickstarts/manage-data/use-dbt-cloud-with-teradata-vantage.md deleted file mode 100644 index 56bca7a702b..00000000000 --- a/quickstarts/manage-data/use-dbt-cloud-with-teradata-vantage.md +++ /dev/null @@ -1,147 +0,0 @@ ---- -id: use-dbt-cloud-with-teradata-vantage -sidebar_position: 4.4 -author: Mohan Talla -email: mohan.talla@teradata.com -description: Use dbt Cloud (data build tool) with Teradata Vantage. -keywords: [dbt Cloud, data warehouses, compute storage separation, teradata, vantage, cloud data platform, object storage, business intelligence, enterprise analytics, elt, dbt.] ---- - -import TrialDocsNote from '../_partials/teradata_trial.mdx' - -# dbt Cloud with Teradata Vantage - -This tutorial demonstrates how to use dbt Cloud with Teradata Vantage. It's based on the original [dbt Jaffle Shop tutorial](https://github.com/Teradata/jaffle_shop-dev). A couple of models have been adjusted to the SQL dialect supported by Vantage. - -## Prerequisites - -* You have a [dbt Cloud account](https://www.getdbt.com/signup/). -* Access to a Teradata Vantage instance. - - - -### About the Jaffle Shop warehouse - -`jaffle_shop` is a fictional e-commerce store. This dbt project transforms raw data from an app database into a dimensional model with customer and order data ready for analytics. - -The raw data from the app consists of customers, orders, and payments, with the following entity-relationship diagram: - -![](../images/dbt1.svg) - -dbt takes these raw data table and builds the following dimensional model, which is more suitable for analytics tools: - -![](../images/dbt2.svg) - -## Creating a project in dbt Cloud and connect to a Teradata environment -Create a new project in dbt Cloud. -* Login to your dbt Cloud account -* Click on "Account home" -* In the "All projects" list click on "+ New Project" - -![](../images/dbt-cloud/dbt-cloud1.png) - -### dbt Project setup -The setup of a dbt Cloud project includes the following steps: -- "Name your project" -- "Configure your development environment" -- "Setup a repository" - -1. Enter a project name and click "Continue". - - ![](../images/dbt-cloud/dbt-cloud2.png) - -2. In "Configure your development environment", click "Add new connection" and follow the steps below. Once you have Teradata Vantage connections established you can simply select them from the dropdown in future projects. - ![](../images/dbt-cloud/dbt-cloud3.png) - - - Select "Teradata" - ![](../images/dbt-cloud/dbt-cloud4.png) - - - Provide a name for the connection that is meaningful. - - Fill in all the required details in the "Settings" section. - ![](../images/dbt-cloud/dbt-cloud5.png) - - The only required field in the settings is the connection host, if your Vantage Environment is behind a firewall you might need to whitelist the provided IP addresses that dbt Cloud uses in your specific environment. - - - Save the connection. - -3. Once the Teradata connection is created, return to the project setup page of your project, select your connection in the "Configure your developer environment" dropdown, and provide the required "Development Credentials". - - ![](../images/dbt-cloud/dbt-cloud6.png) - -4. Click "Test Connection". - - This verifies that dbt Cloud can access your Teradata database. - - If the connection test succeeds, click Save. - - If the connection fails, verify your Teradata settings and credentials. - - If the issue persists, reach out to Teradata support at support.teradata.com. For dbt-teradata related issues, start a discussion on the dbt-teradata GitHub at https://github.com/Teradata/dbt-teradata. - -## Import a sample dbt project to dbt Cloud -1. Fork the following repository to your github account - https://github.com/Teradata/jaffle_shop-dev - -1. In "Setup a repository" select the "Git Clone" option - -2. Paste the following link in the git URL field, remember to substitute your github handle in order it picks the fork of the sample repository. - ``` - git@github.com:{your github handle}/jaffle_shop-dev.git - ``` - ![](../images/dbt-cloud/dbt-cloud7.png) - - This will generate a "Deploy key", this key needs to be deployed to your github, refer to [this guide](https://docs.getdbt.com/docs/cloud/git/import-a-project-by-git-url) for detailed instructions. - - Once the key is deployed, the project will be ready for further development. - - ![img.png](../images/dbt-cloud/dbt-cloud8.png) - -## Visualize the project on dbt Cloud IDE - -Select "Start developing in the IDE". You will be redirected to the development page of the dbt Cloud IDE. - -![](../images/dbt-cloud/dbt-cloud9.png) -In the "File explorer" section, you can browse through the project. - -## Create an environment for managing staging and production workflows for the project - -### Create a dbt Cloud environment - -Before deploying the project, an environment must be created, for this navigate to the dashboard of your project and click "create environment" -* Determine the project's stage of development and select one of the deployment options: General, Staging, or Production, for this guide you might want to create it as a staging environment. -* for dbt version select versionless. -* From the drop-down menu, choose the previously configured connection. -![](../images/dbt-cloud/dbt-cloud10.png) - -Provide the Deployment credentials for the connection and test the connection. -Once the connection is successful, save this environment. -![](../images/dbt-cloud/dbt-cloud11.png) - -Now we have successfully created the environment for creating jobs. -![](../images/dbt-cloud/dbt-cloud12.png) - -### The next step is to create the jobs - -- Clicking the "Create job" button directs you to the "Deploy job" configuration page. -- Add a job name to identify the job. -- Select your environment from the drop-down menu, -- Choose the job to run, such as `dbt build`, `dbt seed`, etc. - -You can schedule these jobs using the provided checkbox and enable source freshness from the same section. Additionally, advanced configurations such as threads and target name can be adjusted based on the project’s requirements - -![](../images/dbt-cloud/dbt-cloud13.png) - -![](../images/dbt-cloud/dbt-cloud14.png) - -After the job completes, you will be able to view the following: -1. Run summary – Displays the various stages of the job along with their run times. Expanding these summaries provides access to console and debug logs, which can be downloaded. -2. Lineage – Selecting the "Lineage" option at the top displays a graph representing the data flow in your project. -3. Model timing – Shows the execution times of models and tests. -4. Artifacts – Artifacts from your runs, such as the manifest.json file, are saved by dbt Cloud, with download links provided. - -![](../images/dbt-cloud/dbt-cloud15.png) - -## Summary - -This tutorial demonstrates how to use dbt Cloud with Teradata Vantage, adapting the dbt Jaffle Shop example. It covers steps for project creation, environment configuration, and job setup in dbt Cloud with Teradata. - -## Further reading -- Learn more with [dbt Learn courses](https://learn.getdbt.com) -- [How we provision Teradata Clearscape Vantage instance](https://developers.teradata.com/quickstarts/get-access-to-vantage/clearscape-analytics-experience/getting-started-with-csae/) - \ No newline at end of file diff --git a/quickstarts/manage-data/use-dbt-cloud-with-teradata.md b/quickstarts/manage-data/use-dbt-cloud-with-teradata.md new file mode 100644 index 00000000000..8a95e8de08a --- /dev/null +++ b/quickstarts/manage-data/use-dbt-cloud-with-teradata.md @@ -0,0 +1,186 @@ +--- +id: use-dbt-cloud-with-teradata +sidebar_position: 4.4 +author: Mohan Talla, Vidhan Bhonsle +email: developer.relations@teradata.com +description: Use dbt Cloud (data build tool) with Teradata. +keywords: [dbt Cloud, data warehouses, compute storage separation, teradata, cloud data platform, object storage, business intelligence, enterprise analytics, elt, dbt] +page_last_update: July 6th, 2026 +--- + +import TrialDocsNote from '../_partials/teradata_trial.mdx' + +# dbt Cloud with Teradata + +This tutorial demonstrates how to use dbt Cloud with Teradata. It's based on the original [dbt Jaffle Shop tutorial](https://github.com/Teradata/jaffle_shop-dev). A couple of models have been adjusted to the SQL dialect supported by Teradata. + +## Prerequisites + +- A dbt Cloud account (https://www.getdbt.com/signup/) +- Access to a Teradata instance + + + +### About the Jaffle Shop warehouse + +`jaffle_shop` is a fictional e-commerce store. This dbt project transforms raw data from an app database into a dimensional model with customer and order data ready for analytics. + +The raw data from the app consists of customers, orders, and payments, with the following entity-relationship diagram: + +![Jaffle Shop ER diagram](../images/dbt1.svg) + +dbt takes these raw data tables and builds the following dimensional model, which is more suitable for analytics tools: + +![Dimensional model diagram](../images/dbt2.svg) + +## Creating a project in dbt Cloud and connect to a Teradata environment +Create a new project in dbt Cloud. +* Log in to your dbt Cloud account +* Create a new project. + +### dbt Project setup +The setup of a dbt Cloud project includes the following steps: +- Name your project +- Configure your development environment +- Set up a repository + +1. Enter a project name and click **Continue**. + + ![dbt Cloud - Project name dialog](../images/dbt-cloud/1-create-project.png + ) + +2. In **Configure your development environment**, select a connection. If you do not already have a connection, click **Add new connection**. + + ![dbt Cloud - Add new connection](../images/dbt-cloud/2.1-add-new-connection.png) + +3. Select **Teradata**. + + ![dbt Cloud - Select Teradata connection](../images/dbt-cloud/2.2-add-new-connection.png) + +4. Provide a meaningful connection name and enter the Teradata host in the **Settings** section. + + ![dbt Cloud - Connection settings](../images/dbt-cloud/2.3-settings.png) + + The required field in the settings is the connection host. If your Teradata environment is behind a firewall, you might need to allowlist the IP addresses that dbt Cloud uses in your environment. + +5. Save the connection. Once the Teradata connection is created, return to the project setup page and select the connection in the **Configure your development environment** section. + +6. Provide the required user credentials: + + - Username + - Password + - Schema + +7. Click **Test connection** to verify that dbt Cloud can access your Teradata database. + + If the connection test succeeds, save the configuration and continue. + + If the connection test runs with dbt Fusion and fails with an error similar to `unknown variant teradata`, click **Skip** and continue with the repository setup. You can validate the Teradata connection later while creating the deployment environment by selecting **Latest** as the dbt version. + + ![dbt Cloud - Connection and user credentials](../images/dbt-cloud/2.4-connection-and-credentials.png) + +## Import a sample dbt project to dbt Cloud + +1. Fork the following repository to your GitHub account: + + ``` + https://github.com/Teradata/jaffle_shop-dev + ``` + +2. In **Set up a repository**, select the **Git Clone** option. + +3. Paste the following link in the Git URL field. Replace `{your GitHub handle}` with your GitHub handle so that dbt Cloud picks your fork of the sample repository. + + ``` + git@github.com:{your GitHub handle}/jaffle_shop-dev.git + ``` + ![dbt Cloud - Set up repository](../images/dbt-cloud/3-set-up-repo.png) + + This will generate a deploy key. This key needs to be added to your GitHub repository. Refer to [this guide](https://docs.getdbt.com/docs/cloud/git/import-a-project-by-git-url) for detailed instructions. + +## Create an environment for managing staging and production workflows for the project +Before deploying the project, create a dbt Cloud environment. + +:::note +In some cases, the left navigation menu may not be visible in dbt Cloud. In that case, click your profile or account icon at the bottom-left of the page, then go to **Account settings** > **Credentials** > select your project > **configure your development environment and add a connection**. +::: + +1. From the left navigation, go to **Orchestration** > **Environments**. + + ![dbt Cloud - Navigate to environments](../images/dbt-cloud/4-environment.png) + +2. Click **+ Create environment**. + +3. In **Environment settings**, provide the following details: + + - **Environment name**: Enter a meaningful name, such as `dbt-teradata-env`. + - **Environment type**: Select **Deployment**. + - **Set deployment type**: Select **Staging**. + - **dbt version**: Select **Latest**. + + :::note + For this quickstart, select **Latest** as the dbt version. Do not select a Fusion option. + ::: + + ![dbt Cloud - Environment settings](../images/dbt-cloud/5-env-setting.png) + +4. In **Connection profiles**, click **Assign profile**, and create a new profile. + + ![dbt Cloud - Assign profile](../images/dbt-cloud/6-assign-profile.png) + +5. In **New profile**, provide a profile name and select the Teradata connection created earlier. If the connection is not available, click **Add new connection** and create a Teradata connection. + + ![dbt Cloud - Select Teradata connection profile](../images/dbt-cloud/7-add-new-con.png) + +6. Provide the deployment credentials: + + - Username + - Password + - Schema + +7. Click **Test connection**. Once the connection test completes successfully, create the profile. Then, select the newly created profile from the dropdown and save the environment. + + ![dbt Cloud - Test deployment connection](../images/dbt-cloud/8-test-connection.png) + +Now you have created a dbt Cloud environment for running jobs. + +## Create and run a job + +1. On the environment page, click **Create job**. From the available options, select **Deploy job**. + + ![dbt Cloud - Create job](../images/dbt-cloud/9-create-job.png) + +2. On the **Deploy job** page, provide the job details: + + - **Job name**: Enter a name such as `jaffle_shop`. + - **Environment**: Select the environment created earlier. + - **Commands**: Add the command to run, such as `dbt build`. + + You can schedule the job using the scheduling options. You can also enable source freshness and adjust advanced configurations such as threads and target name based on your project requirements. + + ![dbt Cloud - Deploy job configuration](../images/dbt-cloud/10-deploy-job.png) + +3. Save the job. + +4. After saving the job, dbt Cloud opens the job details page. Click **Run now** to start the job. + + ![dbt Cloud - Run job](../images/dbt-cloud/11-run-job.png) + +5. After the job completes, verify that the run is successful. + + ![dbt Cloud - Successful job run](../images/dbt-cloud/12-final-output.png) + +After the job completes, you will be able to view the following: + +1. **Run summary** – Displays the various stages of the job along with their run times. Expanding these summaries provides access to console and debug logs, which can be downloaded. +2. **Lineage** – Displays a graph representing the data flow in your project. +3. **Model timing** – Shows the execution times of models and tests. +4. **Artifacts** – Provides access to artifacts from your runs, such as the `manifest.json` file. + +## Summary + +This tutorial demonstrates how to use dbt Cloud with Teradata, adapting the dbt Jaffle Shop example. It covers steps for project creation, environment configuration, and job setup in dbt Cloud with Teradata. + +## Further reading + +- Learn more with [dbt Learn courses](https://learn.getdbt.com) \ No newline at end of file