{"id":1129,"date":"2025-12-31T21:36:48","date_gmt":"2025-12-31T21:36:48","guid":{"rendered":"https:\/\/ranaghazzi.com\/?page_id=1129"},"modified":"2026-05-29T01:05:22","modified_gmt":"2026-05-29T01:05:22","slug":"ibm-stocks-pilpline","status":"publish","type":"page","link":"https:\/\/ranaghazzi.com\/?page_id=1129","title":{"rendered":"IBM Stocks &#8211; ETL Pipeline"},"content":{"rendered":"<p><style>\n    .light-font-container, .light-font-container p, .light-font-container h2, .light-font-container li {<br \/>\n        font-weight: #FFFFFF !important;<br \/>\n    }<br \/>\n<\/style>\n<\/p>\n<div class=\"light-font-container\" style=\"background-color: #2b85d9; padding: 40px; border-radius: 15px;\">\n\n\n<div style=\"height:13px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<h2 class=\"wp-block-heading has-text-align-center alignwide is-style-text-display has-contrast-color has-text-color has-background has-link-color has-x-large-font-size wp-elements-3df358166f1a234ca2073aa57949b046 is-style-text-display--1\" style=\"background-color:#2b85d9\"><strong>IBM Stocks ETL &#8211; Silver layer<\/strong><\/h2>\n\n\n\n<p class=\"has-text-align-center has-contrast-color has-text-color has-link-color has-large-font-size wp-elements-f4354f4ab89ef11fe34ed530a006571f wp-block-paragraph\"><strong>Tools: Databricks | Pyspark | Pandas|<code>&nbsp;Numpy<\/code>&nbsp;|&nbsp;<code>delta.tables<\/code><\/strong><\/p>\n\n\n\n<div class=\"wp-block-buttons alignwide is-content-justification-center is-layout-flex wp-container-core-buttons-is-layout-35f06ea7 wp-block-buttons-is-layout-flex\">\n<div class=\"wp-block-button\"><a class=\"wp-block-button__link has-contrast-color has-text-color has-background has-link-color has-large-font-size has-custom-font-size wp-element-button\" href=\"https:\/\/github.com\/Ranoush-USA\/Databricks_IBM\/blob\/first_upload\/IBM\/resources\/notebooks\/IBM_silver.ipynb.ipynb\" style=\"background-color:#813987b5;padding-right:var(--wp--preset--spacing--50);padding-left:var(--wp--preset--spacing--50)\" target=\"_blank\" rel=\"noreferrer noopener\"><strong>GitHub<\/strong><\/a><\/div>\n\n\n\n<div class=\"wp-block-button\"><a class=\"wp-block-button__link has-contrast-color has-text-color has-background has-link-color has-large-font-size has-custom-font-size wp-element-button\" href=\"https:\/\/Ranoush-USA.github.io\/IBM-ETL\/\" style=\"background-color:#c9fa66;padding-right:var(--wp--preset--spacing--50);padding-left:var(--wp--preset--spacing--50)\" target=\"_blank\" rel=\"noreferrer noopener\"><strong>Digram<\/strong><\/a><\/div>\n\n\n<\/div>\n\n\n\n<div style=\"height:38px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-group has-background has-global-padding is-layout-constrained wp-block-group-is-layout-constrained\" style=\"background-color:#2b85d9\">\n<h2 class=\"wp-block-heading\"><strong>Description :<\/strong><\/h2>\n\n\n\n<div style=\"height:20px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"has-contrast-color has-text-color has-background has-link-color has-large-font-size wp-elements-9fd0175a4608c10cb993f1d23f4dd5e7 wp-block-paragraph\" style=\"background-color:#2b86d9a1;padding-right:var(--wp--preset--spacing--60);padding-left:var(--wp--preset--spacing--60)\">The <strong>Silver layer<\/strong> takes the Bronze data and produces a clean, curated, always-current version of each record. It applies type casting, null validation, and deduplication, then uses a Delta MERGE (upsert) operation keyed on the date field \u2014 updating existing records if they&#8217;ve changed and inserting new ones if they haven&#8217;t been seen before.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">       <strong>Task 3: Silver_Merge<\/strong><a href=\"https:\/\/github.com\/Ranoush-USA\/Databricks_IBM\/tree\/first_upload\/IBM#task-3-silver_merge\"><\/a><\/h2>\n\n\n\n<p class=\"has-contrast-color has-text-color has-background has-link-color has-large-font-size wp-elements-01abd6137811d8080caeb2e7b196daba wp-block-paragraph\" style=\"background-color:#2b86d9a1;padding-right:var(--wp--preset--spacing--60);padding-left:var(--wp--preset--spacing--60)\"><strong>Notebook:<\/strong>&nbsp;IBM _silver<br><strong>Dependencies:<\/strong>&nbsp;Waits for Auto_Loader_bronze to complete<\/p>\n\n\n\n<h1 class=\"wp-block-heading has-contrast-color has-text-color has-background has-link-color has-large-font-size wp-elements-934c10c0d4f02342d259baa117d8b19c\" style=\"background-color:#2b86d9a1\"><strong>What it does:<\/strong><\/h1>\n\n\n\n<ol style=\"background-color:#2b86d9a1;padding-right:var(--wp--preset--spacing--60);padding-left:var(--wp--preset--spacing--60)\" class=\"wp-block-list has-contrast-color has-text-color has-background has-link-color has-large-font-size wp-elements-e3ba06f3abce77ef12f410634cef6199\">\n<li>Loads configuration from\u00a0config_Parms<\/li>\n\n\n\n<li>Creates silver table schema if it doesn&#8217;t exist<\/li>\n\n\n\n<li>Reads from\u00a0workspace.bronze.ibm<\/li>\n\n\n\n<li><strong>Data Quality Transformations:<\/strong>\n<ul class=\"wp-block-list\">\n<li>Casts string columns to proper types (DATE, DOUBLE, BIGINT, TIMESTAMP)<\/li>\n\n\n\n<li>Drops null values on critical columns (Date, Open, High, Low, Close, Volume)<\/li>\n\n\n\n<li>Deduplicates by Date (keeps latest record per date)<\/li>\n\n\n\n<li>Filters invalid data (Close > 0)<\/li>\n\n\n\n<li>Adds\u00a0ingested_at\u00a0timestamp<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li><strong>MERGE operation<\/strong>\u00a0into\u00a0workspace.silver.ibm:\n<ul class=\"wp-block-list\">\n<li>If Date exists: UPDATE all columns<\/li>\n\n\n\n<li>If Date is new: INSERT new row<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>Result: Clean, typed, deduplicated data ready for analytics<\/li>\n<\/ol>\n\n\n\n<p class=\"has-contrast-color has-text-color has-background has-link-color has-large-font-size wp-elements-8ee865404a625ed23f0fa180e0fa9aee wp-block-paragraph\" style=\"background-color:#2b86d9a1;padding-right:var(--wp--preset--spacing--60);padding-left:var(--wp--preset--spacing--60)\"><strong>Output:<\/strong>\u00a0The end result is a reliable, incrementally-updated data foundation ready to power business intelligence tools and real-time dashboards with accurate, deduplicated IBM stock history.<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n<\/div>\n\n\n\n<hr class=\"wp-block-separator has-text-color has-alpha-channel-opacity has-background has-link-color\" style=\"background-color:#2b86d9a1;color:#2b86d9a1\"\/>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<div class=\"wp-block-group has-global-padding is-layout-constrained wp-block-group-is-layout-constrained\"><\/div>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n","protected":false},"excerpt":{"rendered":"<p>IBM Stocks ETL &#8211; Silver layer Tools: Databricks | Pyspark | Pandas|&nbsp;Numpy&nbsp;|&nbsp;delta.tables Description : The Silver layer takes the Bronze data and produces a clean, curated, always-current version of each record. It applies type casting, null validation, and deduplication, then uses a Delta MERGE (upsert) operation keyed on the date field \u2014 updating existing records [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":0,"parent":30,"menu_order":0,"comment_status":"closed","ping_status":"closed","template":"","meta":{"footnotes":""},"class_list":["post-1129","page","type-page","status-publish","hentry"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v27.2 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>IBM Stocks - ETL Pipeline - Rana Nasri Ghazzi<\/title>\n<meta name=\"description\" content=\"Browse real-world data projects by Rana Ghazzi, covering data cleaning, analysis, and storytelling with Python, SQL, and Tableau Explore Rana Ghazzi&#039;s data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI &amp; Python.\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/ranaghazzi.com\/?page_id=1129\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"IBM Stocks - ETL Pipeline - Rana Nasri Ghazzi\" \/>\n<meta property=\"og:description\" content=\"Browse real-world data projects by Rana Ghazzi, covering data cleaning, analysis, and storytelling with Python, SQL, and Tableau Explore Rana Ghazzi&#039;s data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI &amp; Python.\" \/>\n<meta property=\"og:url\" content=\"https:\/\/ranaghazzi.com\/?page_id=1129\" \/>\n<meta property=\"og:site_name\" content=\"Rana Nasri Ghazzi\" \/>\n<meta property=\"article:modified_time\" content=\"2026-05-29T01:05:22+00:00\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data1\" content=\"2 minutes\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/ranaghazzi.com\/?page_id=1129\",\"url\":\"https:\/\/ranaghazzi.com\/?page_id=1129\",\"name\":\"IBM Stocks - ETL Pipeline - Rana Nasri Ghazzi\",\"isPartOf\":{\"@id\":\"https:\/\/ranaghazzi.com\/#website\"},\"datePublished\":\"2025-12-31T21:36:48+00:00\",\"dateModified\":\"2026-05-29T01:05:22+00:00\",\"description\":\"Browse real-world data projects by Rana Ghazzi, covering data cleaning, analysis, and storytelling with Python, SQL, and Tableau Explore Rana Ghazzi's data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI & Python.\",\"breadcrumb\":{\"@id\":\"https:\/\/ranaghazzi.com\/?page_id=1129#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/ranaghazzi.com\/?page_id=1129\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/ranaghazzi.com\/?page_id=1129#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\/\/ranaghazzi.com\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Home\",\"item\":\"https:\/\/ranaghazzi.com\/\"},{\"@type\":\"ListItem\",\"position\":3,\"name\":\"Projects\",\"item\":\"https:\/\/ranaghazzi.com\/?page_id=30\"},{\"@type\":\"ListItem\",\"position\":4,\"name\":\"IBM Stocks &#8211; ETL Pipeline\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/ranaghazzi.com\/#website\",\"url\":\"https:\/\/ranaghazzi.com\/\",\"name\":\"Rana Nasri Ghazzi\",\"description\":\"Turning Data into Decisions\",\"publisher\":{\"@id\":\"https:\/\/ranaghazzi.com\/#\/schema\/person\/d8ee34f53cb0df9faaf816fb5363a4cc\"},\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/ranaghazzi.com\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":[\"Person\",\"Organization\"],\"@id\":\"https:\/\/ranaghazzi.com\/#\/schema\/person\/d8ee34f53cb0df9faaf816fb5363a4cc\",\"name\":\"Rana Ghazzi\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png\",\"url\":\"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png\",\"contentUrl\":\"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png\",\"width\":1024,\"height\":1024,\"caption\":\"Rana Ghazzi\"},\"logo\":{\"@id\":\"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png\"}}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"IBM Stocks - ETL Pipeline - Rana Nasri Ghazzi","description":"Browse real-world data projects by Rana Ghazzi, covering data cleaning, analysis, and storytelling with Python, SQL, and Tableau Explore Rana Ghazzi's data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI & Python.","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/ranaghazzi.com\/?page_id=1129","og_locale":"en_US","og_type":"article","og_title":"IBM Stocks - ETL Pipeline - Rana Nasri Ghazzi","og_description":"Browse real-world data projects by Rana Ghazzi, covering data cleaning, analysis, and storytelling with Python, SQL, and Tableau Explore Rana Ghazzi's data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI & Python.","og_url":"https:\/\/ranaghazzi.com\/?page_id=1129","og_site_name":"Rana Nasri Ghazzi","article_modified_time":"2026-05-29T01:05:22+00:00","twitter_card":"summary_large_image","twitter_misc":{"Est. reading time":"2 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/ranaghazzi.com\/?page_id=1129","url":"https:\/\/ranaghazzi.com\/?page_id=1129","name":"IBM Stocks - ETL Pipeline - Rana Nasri Ghazzi","isPartOf":{"@id":"https:\/\/ranaghazzi.com\/#website"},"datePublished":"2025-12-31T21:36:48+00:00","dateModified":"2026-05-29T01:05:22+00:00","description":"Browse real-world data projects by Rana Ghazzi, covering data cleaning, analysis, and storytelling with Python, SQL, and Tableau Explore Rana Ghazzi's data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI & Python.","breadcrumb":{"@id":"https:\/\/ranaghazzi.com\/?page_id=1129#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/ranaghazzi.com\/?page_id=1129"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/ranaghazzi.com\/?page_id=1129#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/ranaghazzi.com\/"},{"@type":"ListItem","position":2,"name":"Home","item":"https:\/\/ranaghazzi.com\/"},{"@type":"ListItem","position":3,"name":"Projects","item":"https:\/\/ranaghazzi.com\/?page_id=30"},{"@type":"ListItem","position":4,"name":"IBM Stocks &#8211; ETL Pipeline"}]},{"@type":"WebSite","@id":"https:\/\/ranaghazzi.com\/#website","url":"https:\/\/ranaghazzi.com\/","name":"Rana Nasri Ghazzi","description":"Turning Data into Decisions","publisher":{"@id":"https:\/\/ranaghazzi.com\/#\/schema\/person\/d8ee34f53cb0df9faaf816fb5363a4cc"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/ranaghazzi.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":["Person","Organization"],"@id":"https:\/\/ranaghazzi.com\/#\/schema\/person\/d8ee34f53cb0df9faaf816fb5363a4cc","name":"Rana Ghazzi","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png","url":"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png","contentUrl":"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png","width":1024,"height":1024,"caption":"Rana Ghazzi"},"logo":{"@id":"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png"}}]}},"_hostinger_reach_plugin_has_subscription_block":false,"_hostinger_reach_plugin_is_elementor":false,"_links":{"self":[{"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/pages\/1129","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/pages"}],"about":[{"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/types\/page"}],"author":[{"embeddable":true,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1129"}],"version-history":[{"count":243,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/pages\/1129\/revisions"}],"predecessor-version":[{"id":5122,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/pages\/1129\/revisions\/5122"}],"up":[{"embeddable":true,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/pages\/30"}],"wp:attachment":[{"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1129"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}