{"id":5291,"date":"2026-06-04T16:36:46","date_gmt":"2026-06-04T16:36:46","guid":{"rendered":"https:\/\/ranaghazzi.com\/?p=5291"},"modified":"2026-06-04T17:24:03","modified_gmt":"2026-06-04T17:24:03","slug":"postgresql-sql-vs-pandas-vs-pyspark","status":"publish","type":"post","link":"https:\/\/ranaghazzi.com\/?p=5291","title":{"rendered":"PostgreSQL (SQL) vs pandas vs PySpark."},"content":{"rendered":"<p><style>\n    .light-font-container, .light-font-container p, .light-font-container h2, .light-font-container li {<br \/>\n        font-weight: #FFFFFF !important;<br \/>\n    }<br \/>\n<\/style>\n<\/p>\n<div class=\"light-font-container\" style=\"background-color: #FFFFFF; padding: 40px; border-radius: 15px;\">\n\n\n<h2 class=\"wp-block-heading has-text-align-center\">PostgreSQL (SQL) vs pandas vs PySpark.<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Read \/ Inspect<\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Preview rows<\/td><td>SELECT * FROM t LIMIT 5;<\/td><td>df.head(5)<\/td><td>df.show(5)<\/td><\/tr><tr><td>Row count<\/td><td>SELECT count(*) FROM t;<\/td><td>len(df)<\/td><td>df.count()<\/td><\/tr><tr><td>Columns\/schema<\/td><td>\\d t<\/td><td>df.dtypes<\/td><td>df.printSchema()<\/td><\/tr><tr><td>Distinct<\/td><td>SELECT DISTINCT a FROM t;<\/td><td>df.a.drop_duplicates()<\/td><td>df.select(&#8220;a&#8221;).distinct()<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Select \/ Filter<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Select cols<\/td><td>SELECT a, b FROM t;<\/td><td>df[[&#8220;a&#8221;,&#8221;b&#8221;]]<\/td><td>df.select(&#8220;a&#8221;,&#8221;b&#8221;)<\/td><\/tr><tr><td>Filter<\/td><td>WHERE age &gt; 30<\/td><td>df[df.age &gt; 30]<\/td><td>df.filter(F.col(&#8220;age&#8221;) &gt; 30)<\/td><\/tr><tr><td>Multi-condition<\/td><td>WHERE a &gt; 1 AND b &lt; 5<\/td><td>df[(df.a&gt;1) &amp; (df.b&lt;5)]<\/td><td>df.filter((F.col(&#8220;a&#8221;)&gt;1) &amp; (F.col(&#8220;b&#8221;)&lt;5))<\/td><\/tr><tr><td>In list<\/td><td>WHERE c IN (1,2)<\/td><td>df[df.c.isin([1,2])]<\/td><td>df.filter(F.col(&#8220;c&#8221;).isin(1,2))<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Columns \/ Expressions<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>New column<\/td><td>SELECT a + b AS c<\/td><td>df[&#8220;c&#8221;] = df.a + df.b<\/td><td>df.withColumn(&#8220;c&#8221;, F.col(&#8220;a&#8221;)+F.col(&#8220;b&#8221;))<\/td><\/tr><tr><td>Rename<\/td><td>SELECT a AS x<\/td><td>df.rename(columns={&#8220;a&#8221;:&#8221;x&#8221;})<\/td><td>df.withColumnRenamed(&#8220;a&#8221;,&#8221;x&#8221;)<\/td><\/tr><tr><td>Cast<\/td><td>CAST(a AS INT)<\/td><td>df.a.astype(&#8220;int&#8221;)<\/td><td>F.col(&#8220;a&#8221;).cast(&#8220;int&#8221;)<\/td><\/tr><tr><td>Conditional<\/td><td>CASE WHEN a&gt;0 THEN &#8216;pos&#8217; ELSE &#8216;neg&#8217; END<\/td><td>np.where(df.a&gt;0,&#8221;pos&#8221;,&#8221;neg&#8221;)<\/td><td>F.when(F.col(&#8220;a&#8221;)&gt;0,&#8221;pos&#8221;).otherwise(&#8220;neg&#8221;)<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Aggregate \/ Group<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Group + sum<\/td><td>SELECT k, sum(v) FROM t GROUP BY k;<\/td><td>df.groupby(&#8220;k&#8221;).v.sum()<\/td><td>df.groupBy(&#8220;k&#8221;).agg(F.sum(&#8220;v&#8221;))<\/td><\/tr><tr><td>Count per group<\/td><td>SELECT k, count(*) GROUP BY k;<\/td><td>df.groupby(&#8220;k&#8221;).size()<\/td><td>df.groupBy(&#8220;k&#8221;).count()<\/td><\/tr><tr><td>Mean<\/td><td>SELECT avg(v) FROM t;<\/td><td>df.v.mean()<\/td><td>df.agg(F.mean(&#8220;v&#8221;))<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Join \/ Combine<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Left join<\/td><td>a LEFT JOIN b USING(id)<\/td><td>a.merge(b, on=&#8221;id&#8221;, how=&#8221;left&#8221;)<\/td><td>a.join(b, on=&#8221;id&#8221;, how=&#8221;left&#8221;)<\/td><\/tr><tr><td>Stack rows<\/td><td>SELECT * FROM a UNION ALL SELECT * FROM b<\/td><td>pd.concat([a,b])<\/td><td>a.union(b)<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Sort \/ Nulls \/ Output<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Sort desc<\/td><td>ORDER BY a DESC<\/td><td>df.sort_values(&#8220;a&#8221;, ascending=False)<\/td><td>df.orderBy(F.desc(&#8220;a&#8221;))<\/td><\/tr><tr><td>Drop nulls<\/td><td>WHERE a IS NOT NULL<\/td><td>df.dropna()<\/td><td>df.na.drop()<\/td><\/tr><tr><td>Fill nulls<\/td><td>COALESCE(a, 0)<\/td><td>df.fillna(0)<\/td><td>df.na.fill(0)<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\">PySpark assumes from pyspark.sql import functions as F. Key difference: SQL and pandas execute eagerly; Spark is lazy until an action like .show() or .collect().\ue056\ue03b\ue0c1<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Window Functions<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Row number per group<\/td><td>row_number() OVER (PARTITION BY k ORDER BY v DESC)<\/td><td>df.sort_values(&#8220;v&#8221;,ascending=False).groupby(&#8220;k&#8221;).cumcount()+1<\/td><td>F.row_number().over(Window.partitionBy(&#8220;k&#8221;).orderBy(F.desc(&#8220;v&#8221;)))<\/td><\/tr><tr><td>Rank<\/td><td>rank() OVER (PARTITION BY k ORDER BY v)<\/td><td>df.groupby(&#8220;k&#8221;).v.rank(method=&#8221;min&#8221;)<br><\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><tbody><tr><td>Dense rank<\/td><td>dense_rank() OVER (&#8230;)<\/td><td>df.groupby(&#8220;k&#8221;).v.rank(method=&#8221;dense&#8221;)<\/td><td>F.dense_rank().over(w)<\/td><\/tr><tr><td>Lag<\/td><td>lag(v,1) OVER (ORDER BY t)<\/td><td>df.v.shift(1)<\/td><td>F.lag(&#8220;v&#8221;,1).over(Window.orderBy(&#8220;t&#8221;))<\/td><\/tr><tr><td>Lead<\/td><td>lead(v,1) OVER (ORDER BY t)<\/td><td>df.v.shift(-1)<\/td><td>F.lead(&#8220;v&#8221;,1).over(Window.orderBy(&#8220;t&#8221;))<\/td><\/tr><tr><td>Running total<\/td><td>sum(v) OVER (ORDER BY t)<\/td><td>df.v.cumsum()<\/td><td>F.sum(&#8220;v&#8221;).over(Window.orderBy(&#8220;t&#8221;).rowsBetween(Window.unboundedPreceding, 0))<\/td><\/tr><tr><td>Group sum (no collapse)<\/td><td>sum(v) OVER (PARTITION BY k)<\/td><td>df.groupby(&#8220;k&#8221;).v.transform(&#8220;sum&#8221;)<\/td><td>F.sum(&#8220;v&#8221;).over(Window.partitionBy(&#8220;k&#8221;))<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Deduplication<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Drop exact dupes<\/td><td>SELECT DISTINCT * FROM t;<\/td><td>df.drop_duplicates()<\/td><td>df.dropDuplicates()<\/td><\/tr><tr><td>Dupes on subset<\/td><td>DISTINCT ON (a,b)<\/td><td>df.drop_duplicates(subset=[&#8220;a&#8221;,&#8221;b&#8221;])<\/td><td>df.dropDuplicates([&#8220;a&#8221;,&#8221;b&#8221;])<\/td><\/tr><tr><td>Keep latest per key<\/td><td>row_number() OVER (PARTITION BY k ORDER BY ts DESC) = 1<\/td><td>df.sort_values(&#8220;ts&#8221;).drop_duplicates(&#8220;k&#8221;, keep=&#8221;last&#8221;)<\/td><td>filter where row_number().over(w)==1<\/td><\/tr><tr><td>Count dupes<\/td><td>GROUP BY a HAVING count(*) &gt; 1<\/td><td>df[df.duplicated(&#8220;a&#8221;, keep=False)]<\/td><td>df.groupBy(&#8220;a&#8221;).count().filter(F.col(&#8220;count&#8221;)&gt;1)<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\">The &#8220;keep latest per key&#8221; pattern in Spark, fully written:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">python<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>w = Window.partitionBy(\"k\").orderBy(F.desc(\"ts\"))\ndf.withColumn(\"rn\", F.row_number().over(w)).filter(F.col(\"rn\")==1).drop(\"rn\")<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Reshape (pivot \/ melt \/ explode)<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Pivot<\/td><td>crosstab(&#8230;) (tablefunc ext)<\/td><td>df.pivot_table(index=&#8221;k&#8221;, columns=&#8221;c&#8221;, values=&#8221;v&#8221;)<\/td><td>df.groupBy(&#8220;k&#8221;).pivot(&#8220;c&#8221;).agg(F.sum(&#8220;v&#8221;))<\/td><\/tr><tr><td>Melt \/ unpivot<\/td><td>UNNEST \/ manual UNION<\/td><td>df.melt(id_vars=&#8221;k&#8221;)<\/td><td>df.select(&#8220;k&#8221;, F.expr(&#8220;stack(&#8230;)&#8221;))<\/td><\/tr><tr><td>Explode array \u2192 rows<\/td><td>unnest(arr)<\/td><td>df.explode(&#8220;arr&#8221;)<\/td><td>df.withColumn(&#8220;x&#8221;, F.explode(&#8220;arr&#8221;))<\/td><\/tr><tr><td>Split string \u2192 array<\/td><td>string_to_array(s, &#8216;,&#8217;)<\/td><td>df.s.str.split(&#8220;,&#8221;)<\/td><td>F.split(F.col(&#8220;s&#8221;), &#8220;,&#8221;)<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>String \/ Date manipulation<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Lowercase<\/td><td>lower(s)<\/td><td>df.s.str.lower()<\/td><td>F.lower(&#8220;s&#8221;)<\/td><\/tr><tr><td>Substring<\/td><td>substring(s,1,3)<\/td><td>df.s.str[0:3]<\/td><td>F.substring(&#8220;s&#8221;,1,3)<\/td><\/tr><tr><td>Concat<\/td><td>a || b<\/td><td>df.a + df.b<\/td><td>F.concat(&#8220;a&#8221;,&#8221;b&#8221;)<\/td><\/tr><tr><td>Replace<\/td><td>replace(s,&#8217;x&#8217;,&#8217;y&#8217;)<\/td><td>df.s.str.replace(&#8220;x&#8221;,&#8221;y&#8221;)<\/td><td>F.regexp_replace(&#8220;s&#8221;,&#8221;x&#8221;,&#8221;y&#8221;)<\/td><\/tr><tr><td>Parse date<\/td><td>to_date(s,&#8217;YYYY-MM-DD&#8217;)<\/td><td>pd.to_datetime(df.s)<\/td><td>F.to_date(&#8220;s&#8221;,&#8221;yyyy-MM-dd&#8221;)<\/td><\/tr><tr><td>Extract year<\/td><td>extract(year from d)<\/td><td>df.d.dt.year<\/td><td>F.year(&#8220;d&#8221;)<\/td><\/tr><tr><td>Date diff (days)<\/td><td>d2 &#8211; d1<\/td><td>(df.d2 &#8211; df.d1).dt.days<\/td><td>F.datediff(&#8220;d2&#8243;,&#8221;d1&#8221;)<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Aggregate edge cases<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Count distinct<\/td><td>count(DISTINCT a)<\/td><td>df.a.nunique()<\/td><td>F.countDistinct(&#8220;a&#8221;)<\/td><\/tr><tr><td>Conditional count<\/td><td>count(*) FILTER (WHERE a>0)<\/td><td>(df.a>0).sum()<\/td><td>F.sum(F.when(F.col(&#8220;a&#8221;)>0,1).otherwise(0))<\/td><\/tr><tr><td>Collect into list<\/td><td>array_agg(v)<\/td><td>df.groupby(&#8220;k&#8221;).v.apply(list)<\/td><td>F.collect_list(&#8220;v&#8221;)<\/td><\/tr><tr><td>First\/last<\/td><td>first_value\/last_value OVER (&#8230;)<\/td><td>df.groupby(&#8220;k&#8221;).v.first()<\/td><td>F.first(&#8220;v&#8221;) \/ F.last(&#8220;v&#8221;)<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Creating a schema \/ defining types<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>PostgreSQL \u2014 types live in the table definition:<\/strong><\/p>\n\n\n\n<p class=\"has-accent-5-background-color has-background has-medium-font-size wp-block-paragraph\">CREATE TABLE users (<br>id         INTEGER PRIMARY KEY,<br>name VARCHAR(100) NOT NULL,<br>age      SMALLINT,<br>signup DATE,<br>active    BOOLEAN DEFAULT true<br>);<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>pandas \u2014 schema is a dtype dict, applied at read or after:<\/strong><\/p>\n\n\n\n<p class=\"has-accent-5-background-color has-background has-medium-font-size wp-block-paragraph\">dtypes = {&#8220;id&#8221;:&#8221;int32&#8243;,&#8221;name&#8221;:&#8221;string&#8221;,&#8221;age&#8221;:&#8221;Int16&#8243;,&#8221;active&#8221;:&#8221;boolean&#8221;}<br>df = pd.read_csv(&#8220;f.csv&#8221;, dtype=dtypes, parse_dates=[&#8220;signup&#8221;])<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>PySpark \u2014 an explicit StructType (preferred over inferSchema for production):<\/strong><\/p>\n\n\n\n<div class=\"wp-block-group is-style-default has-accent-5-background-color has-background has-global-padding is-layout-constrained wp-block-group-is-layout-constrained\">\n<p class=\"has-accent-5-background-color has-background has-medium-font-size wp-block-paragraph\">from pyspark.sql.types import (StructType, StructField, IntegerType,<br>StringType, ShortType, DateType, BooleanType)<\/p>\n\n\n\n<p class=\"has-accent-5-background-color has-background has-medium-font-size wp-block-paragraph\">schema = StructType([<br>StructField(&#8220;id&#8221;, IntegerType(), nullable=False),<br>StructField(&#8220;name&#8221;, StringType(), nullable=False),<br>StructField(&#8220;age&#8221;, ShortType(), nullable=True),<br>StructField(&#8220;signup&#8221;, DateType(), nullable=True),<br>StructField(&#8220;active&#8221;, BooleanType(), nullable=True),<br>])<br>df = spark.read.csv(&#8220;f.csv&#8221;, header=True, schema=schema)<\/p>\n<\/div>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Assigning \/ changing a data type<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Cast a column<\/td><td>CAST(a AS INTEGER) or a::int<\/td><td>df.a.astype(&#8220;int32&#8221;)<\/td><td>df.withColumn(&#8220;a&#8221;, F.col(&#8220;a&#8221;).cast(&#8220;int&#8221;))<\/td><\/tr><tr><td>To string<\/td><td>a::text<\/td><td>df.a.astype(&#8220;string&#8221;)<\/td><td>F.col(&#8220;a&#8221;).cast(&#8220;string&#8221;)<\/td><\/tr><tr><td>To date<\/td><td>to_date(a,&#8217;YYYY-MM-DD&#8217;)<\/td><td>pd.to_datetime(df.a)<\/td><td>F.to_date(&#8220;a&#8221;,&#8221;yyyy-MM-dd&#8221;)<\/td><\/tr><tr><td>To numeric (safe)<\/td><td>a::numeric (errors if bad)<\/td><td>pd.to_numeric(df.a, errors=&#8221;coerce&#8221;)<\/td><td>F.col(&#8220;a&#8221;).cast(&#8220;double&#8221;) (bad \u2192 null)<\/td><\/tr><tr><td>Alter stored type<\/td><td>ALTER TABLE t ALTER COLUMN a TYPE int;<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Data quality checks<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Check<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Null count per col<\/td><td>count(*) &#8211; count(a)<\/td><td>df.a.isna().sum()<\/td><td>df.filter(F.col(&#8220;a&#8221;).isNull()).count()<\/td><\/tr><tr><td>Duplicate keys<\/td><td>GROUP BY id HAVING count(*)>1<\/td><td>df.duplicated(&#8220;id&#8221;).sum()<\/td><td>df.groupBy(&#8220;id&#8221;).count().filter(F.col(&#8220;count&#8221;)>1)<\/td><\/tr><tr><td>Out-of-range<\/td><td>WHERE age &lt; 0 OR age > 120<\/td><td>`df[(df.age&lt;0)<\/td><td>(df.age&gt;120)]`<\/td><\/tr><tr><td>Distinct count<\/td><td>count(DISTINCT a)<\/td><td>df.a.nunique()<\/td><td>df.select(&#8220;a&#8221;).distinct().count()<\/td><\/tr><tr><td>Pattern \/ format<\/td><td>a ~ &#8216;^[0-9]+$&#8217;<\/td><td>df.a.str.match(r&#8221;^\\d+$&#8221;)<\/td><td>F.col(&#8220;a&#8221;).rlike(&#8220;^[0-9]+$&#8221;)<\/td><\/tr><tr><td>Cast-failure rows<\/td><td>\u2014<\/td><td>rows where to_numeric(coerce) is null but original wasn&#8217;t<\/td><td>same: compare pre\/post-cast nulls<\/td><\/tr><tr><td>Enforce at write<\/td><td>CHECK, NOT NULL, UNIQUE constraints<\/td><td>manual assert<\/td><td>manual filter \/ framework<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>A compact null-profile across all columns:<\/strong><\/p>\n\n\n\n<div class=\"wp-block-group has-accent-5-background-color has-background has-global-padding is-layout-constrained wp-block-group-is-layout-constrained\">\n<h1 class=\"wp-block-heading has-accent-5-background-color has-background has-medium-font-size\">pandas<\/h1>\n\n\n\n<p class=\"has-accent-5-background-color has-background has-medium-font-size wp-block-paragraph\">df.isna().sum()<\/p>\n\n\n\n<h1 class=\"wp-block-heading has-accent-5-background-color has-background has-medium-font-size\">PySpark<\/h1>\n\n\n\n<p class=\"has-accent-5-background-color has-background has-medium-font-size wp-block-paragraph\">df.select([F.sum(F.col(c).isNull().cast(&#8220;int&#8221;)).alias(c) for c in df.columns])<\/p>\n<\/div>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Renaming a field:<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table has-medium-font-size\"><table class=\"has-fixed-layout\"><thead><tr><th>Task<\/th><th>PostgreSQL<\/th><th>pandas<\/th><th>PySpark<\/th><\/tr><\/thead><tbody><tr><td>Rename one column<\/td><td>ALTER TABLE t RENAME COLUMN a TO x;<\/td><td>df.rename(columns={&#8220;a&#8221;:&#8221;x&#8221;})<\/td><td>df.withColumnRenamed(&#8220;a&#8221;,&#8221;x&#8221;)<\/td><\/tr><tr><td>Rename several<\/td><td>run multiple RENAME statements<\/td><td>df.rename(columns={&#8220;a&#8221;:&#8221;x&#8221;,&#8221;b&#8221;:&#8221;y&#8221;})<\/td><td>chain .withColumnRenamed(&#8230;) or use <code>select<\/code> with aliases<\/td><\/tr><tr><td>Rename in a query (alias only)<\/td><td>SELECT a AS x FROM t;<\/td><td>n\/a<\/td><td>df.select(F.col(&#8220;a&#8221;).alias(&#8220;x&#8221;))<\/td><\/tr><tr><td>Rename all at once<\/td><td>\u2014<\/td><td>df.columns = [&#8220;x&#8221;,&#8221;y&#8221;,&#8221;z&#8221;]<\/td><td>df.toDF(&#8220;x&#8221;,&#8221;y&#8221;,&#8221;z&#8221;)<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Rename several in Spark:<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">python<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>mapping = {\"a\":\"x\", \"b\":\"y\"}\nfor old, new in mapping.items():\n    df = df.withColumnRenamed(old, new)<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">The structural difference: PostgreSQL enforces schema and quality declaratively (constraints reject bad data on insert), while pandas and Spark are permissive \u2014 types are assigned but nothing stops bad values, so quality has to be checked explicitly after load.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Key distinction: the Postgres ALTER TABLE permanently changes the stored table, while SELECT a AS x only relabels in the query output. pandas rename returns a new frame (use inplace=True or reassign), and Spark always returns a new immutable DataFrame.\ue056\ue03b\ue0c1<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The big conceptual split stays the same: SQL and pandas evaluate eagerly, while Spark builds a lazy plan \u2014 window specs and transformations are just descriptions until an action triggers execution.<\/p>\n","protected":false},"excerpt":{"rendered":"<p>PostgreSQL (SQL) vs pandas vs PySpark. Read \/ Inspect Task PostgreSQL pandas PySpark Preview rows SELECT * FROM t LIMIT 5; df.head(5) df.show(5) Row count SELECT count(*) FROM t; len(df) df.count() Columns\/schema \\d t df.dtypes df.printSchema() Distinct SELECT DISTINCT a FROM t; df.a.drop_duplicates() df.select(&#8220;a&#8221;).distinct() Select \/ Filter Task PostgreSQL pandas PySpark Select cols SELECT a, [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-5291","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v28.0 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>PostgreSQL (SQL) vs pandas vs PySpark. - Rana Nasri Ghazzi<\/title>\n<meta name=\"description\" content=\"Explore Rana Ghazzi&#039;s data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI &amp; Python.\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/ranaghazzi.com\/?p=5291\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"PostgreSQL (SQL) vs pandas vs PySpark. - Rana Nasri Ghazzi\" \/>\n<meta property=\"og:description\" content=\"Explore Rana Ghazzi&#039;s data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI &amp; Python.\" \/>\n<meta property=\"og:url\" content=\"https:\/\/ranaghazzi.com\/?p=5291\" \/>\n<meta property=\"og:site_name\" content=\"Rana Nasri Ghazzi\" \/>\n<meta property=\"article:published_time\" content=\"2026-06-04T16:36:46+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2026-06-04T17:24:03+00:00\" \/>\n<meta name=\"author\" content=\"Rana Ghazzi\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"Written by\" \/>\n\t<meta name=\"twitter:data1\" content=\"Rana Ghazzi\" \/>\n\t<meta name=\"twitter:label2\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data2\" content=\"7 minutes\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/?p=5291#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/?p=5291\"},\"author\":{\"name\":\"Rana Ghazzi\",\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/#\\\/schema\\\/person\\\/d8ee34f53cb0df9faaf816fb5363a4cc\"},\"headline\":\"PostgreSQL (SQL) vs pandas vs PySpark.\",\"datePublished\":\"2026-06-04T16:36:46+00:00\",\"dateModified\":\"2026-06-04T17:24:03+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/?p=5291\"},\"wordCount\":1440,\"commentCount\":0,\"publisher\":{\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/#\\\/schema\\\/person\\\/d8ee34f53cb0df9faaf816fb5363a4cc\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"CommentAction\",\"name\":\"Comment\",\"target\":[\"https:\\\/\\\/ranaghazzi.com\\\/?p=5291#respond\"]}]},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/?p=5291\",\"url\":\"https:\\\/\\\/ranaghazzi.com\\\/?p=5291\",\"name\":\"PostgreSQL (SQL) vs pandas vs PySpark. - Rana Nasri Ghazzi\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/#website\"},\"datePublished\":\"2026-06-04T16:36:46+00:00\",\"dateModified\":\"2026-06-04T17:24:03+00:00\",\"description\":\"Explore Rana Ghazzi's data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI & Python.\",\"breadcrumb\":{\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/?p=5291#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/ranaghazzi.com\\\/?p=5291\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/?p=5291#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\\\/\\\/ranaghazzi.com\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"PostgreSQL (SQL) vs pandas vs PySpark.\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/#website\",\"url\":\"https:\\\/\\\/ranaghazzi.com\\\/\",\"name\":\"Rana Nasri Ghazzi\",\"description\":\"Turning Data into Decisions\",\"publisher\":{\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/#\\\/schema\\\/person\\\/d8ee34f53cb0df9faaf816fb5363a4cc\"},\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/ranaghazzi.com\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":[\"Person\",\"Organization\"],\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/#\\\/schema\\\/person\\\/d8ee34f53cb0df9faaf816fb5363a4cc\",\"name\":\"Rana Ghazzi\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/wp-content\\\/uploads\\\/2025\\\/11\\\/logo.png\",\"url\":\"https:\\\/\\\/ranaghazzi.com\\\/wp-content\\\/uploads\\\/2025\\\/11\\\/logo.png\",\"contentUrl\":\"https:\\\/\\\/ranaghazzi.com\\\/wp-content\\\/uploads\\\/2025\\\/11\\\/logo.png\",\"width\":1024,\"height\":1024,\"caption\":\"Rana Ghazzi\"},\"logo\":{\"@id\":\"https:\\\/\\\/ranaghazzi.com\\\/wp-content\\\/uploads\\\/2025\\\/11\\\/logo.png\"},\"url\":\"https:\\\/\\\/ranaghazzi.com\\\/?author=2\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"PostgreSQL (SQL) vs pandas vs PySpark. - Rana Nasri Ghazzi","description":"Explore Rana Ghazzi's data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI & Python.","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/ranaghazzi.com\/?p=5291","og_locale":"en_US","og_type":"article","og_title":"PostgreSQL (SQL) vs pandas vs PySpark. - Rana Nasri Ghazzi","og_description":"Explore Rana Ghazzi's data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI & Python.","og_url":"https:\/\/ranaghazzi.com\/?p=5291","og_site_name":"Rana Nasri Ghazzi","article_published_time":"2026-06-04T16:36:46+00:00","article_modified_time":"2026-06-04T17:24:03+00:00","author":"Rana Ghazzi","twitter_card":"summary_large_image","twitter_misc":{"Written by":"Rana Ghazzi","Est. reading time":"7 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/ranaghazzi.com\/?p=5291#article","isPartOf":{"@id":"https:\/\/ranaghazzi.com\/?p=5291"},"author":{"name":"Rana Ghazzi","@id":"https:\/\/ranaghazzi.com\/#\/schema\/person\/d8ee34f53cb0df9faaf816fb5363a4cc"},"headline":"PostgreSQL (SQL) vs pandas vs PySpark.","datePublished":"2026-06-04T16:36:46+00:00","dateModified":"2026-06-04T17:24:03+00:00","mainEntityOfPage":{"@id":"https:\/\/ranaghazzi.com\/?p=5291"},"wordCount":1440,"commentCount":0,"publisher":{"@id":"https:\/\/ranaghazzi.com\/#\/schema\/person\/d8ee34f53cb0df9faaf816fb5363a4cc"},"inLanguage":"en-US","potentialAction":[{"@type":"CommentAction","name":"Comment","target":["https:\/\/ranaghazzi.com\/?p=5291#respond"]}]},{"@type":"WebPage","@id":"https:\/\/ranaghazzi.com\/?p=5291","url":"https:\/\/ranaghazzi.com\/?p=5291","name":"PostgreSQL (SQL) vs pandas vs PySpark. - Rana Nasri Ghazzi","isPartOf":{"@id":"https:\/\/ranaghazzi.com\/#website"},"datePublished":"2026-06-04T16:36:46+00:00","dateModified":"2026-06-04T17:24:03+00:00","description":"Explore Rana Ghazzi's data analytics portfolio \u2014 dashboards, visualizations, and insights built with Tableau, Power BI & Python.","breadcrumb":{"@id":"https:\/\/ranaghazzi.com\/?p=5291#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/ranaghazzi.com\/?p=5291"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/ranaghazzi.com\/?p=5291#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/ranaghazzi.com\/"},{"@type":"ListItem","position":2,"name":"PostgreSQL (SQL) vs pandas vs PySpark."}]},{"@type":"WebSite","@id":"https:\/\/ranaghazzi.com\/#website","url":"https:\/\/ranaghazzi.com\/","name":"Rana Nasri Ghazzi","description":"Turning Data into Decisions","publisher":{"@id":"https:\/\/ranaghazzi.com\/#\/schema\/person\/d8ee34f53cb0df9faaf816fb5363a4cc"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/ranaghazzi.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":["Person","Organization"],"@id":"https:\/\/ranaghazzi.com\/#\/schema\/person\/d8ee34f53cb0df9faaf816fb5363a4cc","name":"Rana Ghazzi","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png","url":"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png","contentUrl":"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png","width":1024,"height":1024,"caption":"Rana Ghazzi"},"logo":{"@id":"https:\/\/ranaghazzi.com\/wp-content\/uploads\/2025\/11\/logo.png"},"url":"https:\/\/ranaghazzi.com\/?author=2"}]}},"_links":{"self":[{"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/posts\/5291","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=5291"}],"version-history":[{"count":27,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/posts\/5291\/revisions"}],"predecessor-version":[{"id":5336,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=\/wp\/v2\/posts\/5291\/revisions\/5336"}],"wp:attachment":[{"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=5291"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=5291"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/ranaghazzi.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=5291"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}