[{"data":1,"prerenderedAt":534},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fconnections\u002Fsnowflake-cdc":3},{"id":4,"title":5,"body":6,"description":525,"extension":526,"meta":527,"navigation":528,"path":529,"redirect":530,"seo":531,"stem":532,"__hash__":533},"docs\u002Fdocs\u002Fconnections\u002Fsnowflake-cdc.md","Snowflake CDC",{"type":7,"value":8,"toc":519},"minimark",[9,14,28,37,42,156,163,268,273,278,283,292,302,306,358,362,379,385,408,418,425,479,486,490,515],[10,11,13],"h1",{"id":12},"cdc-into-snowflake","CDC into Snowflake",[15,16,17,18,22,23,27],"p",{},"Snowflake's ingestion is append-only. A bulk load is a ",[19,20,21],"code",{},"COPY INTO",", a change\nstream is ",[24,25,26],"strong",{},"Snowpipe Streaming",", and neither can express an update or a delete —\nonly one more row, appended. CDC therefore records what happened and applies it\nin a second step: changes stream in through Snowpipe Streaming, which is billed\nby the volume ingested rather than by warehouse uptime, and a Snowflake task\nmerges them into the table you query.",[15,29,30,31,36],{},"Sources are MySQL and PostgreSQL; Snowflake is a target only. Setting up the\nconnection itself — account, key pair, role and warehouse — is on the\n",[32,33,35],"a",{"href":34},"\u002Fdocs\u002Fconnections\u002Fsnowflake","Snowflake target"," page, and CDC needs that\nconnection to use key-pair authentication.",[15,38,39],{},[24,40,41],{},"CDC mode:",[43,44,49],"pre",{"className":45,"code":46,"language":47,"meta":48,"style":48},"language-mermaid shiki shiki-themes github-light github-dark","flowchart TB\n    SRC[(\"MySQL binlog\u003Cbr\u002F>PostgreSQL WAL\")]\n    W[\"Writers\"]\n    CH([\"Snowpipe channel\"])\n    P[\"ORDERS__CHANGES_PIPE\u003Cbr\u002F>says where arriving rows go\"]\n    H[(\"ORDERS__CHANGES\u003Cbr\u002F>every change, appended\")]\n    S([\"ORDERS__CHANGES_STREAM\u003Cbr\u002F>what is new since the last run\"])\n    T[\"ORDERS__APPLY\u003Cbr\u002F>a task, on your schedule\"]\n    C[(\"ORDERS\u003Cbr\u002F>what you query\")]\n\n    SRC --> W\n    W -->|\"changes, as they happen\"| CH\n    CH --> P\n    P --> H\n    H --> S\n    S --> T\n    T -->|\"MERGE by primary key\"| C\n","mermaid","",[19,50,51,59,65,71,77,83,89,95,101,107,114,120,126,132,138,144,150],{"__ignoreMap":48},[52,53,56],"span",{"class":54,"line":55},"line",1,[52,57,58],{},"flowchart TB\n",[52,60,62],{"class":54,"line":61},2,[52,63,64],{},"    SRC[(\"MySQL binlog\u003Cbr\u002F>PostgreSQL WAL\")]\n",[52,66,68],{"class":54,"line":67},3,[52,69,70],{},"    W[\"Writers\"]\n",[52,72,74],{"class":54,"line":73},4,[52,75,76],{},"    CH([\"Snowpipe channel\"])\n",[52,78,80],{"class":54,"line":79},5,[52,81,82],{},"    P[\"ORDERS__CHANGES_PIPE\u003Cbr\u002F>says where arriving rows go\"]\n",[52,84,86],{"class":54,"line":85},6,[52,87,88],{},"    H[(\"ORDERS__CHANGES\u003Cbr\u002F>every change, appended\")]\n",[52,90,92],{"class":54,"line":91},7,[52,93,94],{},"    S([\"ORDERS__CHANGES_STREAM\u003Cbr\u002F>what is new since the last run\"])\n",[52,96,98],{"class":54,"line":97},8,[52,99,100],{},"    T[\"ORDERS__APPLY\u003Cbr\u002F>a task, on your schedule\"]\n",[52,102,104],{"class":54,"line":103},9,[52,105,106],{},"    C[(\"ORDERS\u003Cbr\u002F>what you query\")]\n",[52,108,110],{"class":54,"line":109},10,[52,111,113],{"emptyLinePlaceholder":112},true,"\n",[52,115,117],{"class":54,"line":116},11,[52,118,119],{},"    SRC --> W\n",[52,121,123],{"class":54,"line":122},12,[52,124,125],{},"    W -->|\"changes, as they happen\"| CH\n",[52,127,129],{"class":54,"line":128},13,[52,130,131],{},"    CH --> P\n",[52,133,135],{"class":54,"line":134},14,[52,136,137],{},"    P --> H\n",[52,139,141],{"class":54,"line":140},15,[52,142,143],{},"    H --> S\n",[52,145,147],{"class":54,"line":146},16,[52,148,149],{},"    S --> T\n",[52,151,153],{"class":54,"line":152},17,[52,154,155],{},"    T -->|\"MERGE by primary key\"| C\n",[15,157,158,159,162],{},"Nothing is staged and nothing is uploaded: changes go straight into Snowflake as\nthey happen. For a source table ",[19,160,161],{},"ORDERS",", a CDC stream creates five objects, of\nwhich two hold rows:",[164,165,166,179],"table",{},[167,168,169],"thead",{},[170,171,172,176],"tr",{},[173,174,175],"th",{},"Object",[173,177,178],{},"What it is",[180,181,182,221,235,245,259],"tbody",{},[170,183,184,190],{},[185,186,187],"td",{},[19,188,189],{},"ORDERS__CHANGES",[185,191,192,193,196,197,200,201,204,205,208,209,212,213,216,217,220],{},"Every change, appended: the source columns plus ",[19,194,195],{},"_DBC_OP"," (",[19,198,199],{},"I",", ",[19,202,203],{},"U"," or ",[19,206,207],{},"D","), ",[19,210,211],{},"_DBC_SEQ"," (the position in the source log), ",[19,214,215],{},"_DBC_ORD"," (the order changes arrived, which separates several changes sharing one log position) and ",[19,218,219],{},"_DBC_TS",".",[170,222,223,228],{},[185,224,225],{},[19,226,227],{},"ORDERS__CHANGES_PIPE",[185,229,230,231,234],{},"The pipe that feeds it — a single ",[19,232,233],{},"COPY INTO ... FROM TABLE (DATA_SOURCE(TYPE => 'STREAMING'))"," statement that says where arriving rows go. It stores nothing.",[170,236,237,242],{},[185,238,239],{},[19,240,241],{},"ORDERS__CHANGES_STREAM",[185,243,244],{},"What has arrived since the changes were last applied. It stores nothing either; it is a position.",[170,246,247,252],{},[185,248,249],{},[19,250,251],{},"ORDERS__APPLY",[185,253,254,255,258],{},"A Snowflake task carrying the ",[19,256,257],{},"MERGE",", run on the schedule you choose.",[170,260,261,265],{},[185,262,263],{},[19,264,161],{},[185,266,267],{},"An ordinary table holding the latest version of each row, with deleted rows removed. This is the table you query.",[15,269,270,272],{},[19,271,161],{}," is a plain table and holds its own rows — not a view, and nothing\nderived. The changes beside it are there when you need to see how a row got that\nway.",[274,275,277],"h2",{"id":276},"the-changes-are-yours-to-manage","The changes are yours to manage",[15,279,280,282],{},[19,281,189],{}," grows for the life of the stream: every superseded version of\nevery row stays in it, and there is no retention setting yet.",[15,284,285,288,289,291],{},[24,286,287],{},"Changes that have been applied are yours to delete."," Trim them or empty the\ntable entirely, and ",[19,290,161],{}," keeps every row — it holds its own data and is not\nderived from the history. What you lose is the record of how rows reached their\ncurrent values, and nothing else.",[15,293,294,295,298,299,301],{},"Changes that have ",[24,296,297],{},"not"," been applied yet are a different matter: the task\napplies what the stream has not yet consumed, so deleting those rows before it\nruns means they never reach ",[19,300,161],{},". Empty the changes table after an apply, not\nbetween one and the next.",[274,303,305],{"id":304},"what-cdc-into-snowflake-requires","What CDC into Snowflake requires",[307,308,309,316,322],"ul",{},[310,311,312,315],"li",{},[24,313,314],{},"Key-pair authentication."," Snowpipe Streaming accepts a signed key and\nneither a password nor an access token. A stream on a password connection is\nrefused before it starts. New Snowflake connections are created with a key pair\nfor this reason; a connection already saved with a password keeps it and can\nstill be a load target until Snowflake stops accepting passwords from service\nusers during 2026.",[310,317,318,321],{},[24,319,320],{},"A primary key on every replicated table."," Without one there is nothing to\ncollapse the changes by, and the stream stops with that explanation.",[310,323,324,330,331,348,351,352,354,355,357],{},[24,325,326,329],{},[19,327,328],{},"EXECUTE TASK"," on the account."," The task that applies changes cannot run\nwithout it, and it is an account-level privilege the connecting role usually\ncannot grant itself. An account administrator runs this once:",[43,332,336],{"className":333,"code":334,"language":335,"meta":48,"style":48},"language-sql shiki shiki-themes github-light github-dark","USE ROLE ACCOUNTADMIN;\nGRANT EXECUTE TASK ON ACCOUNT TO ROLE \u003Cthe role your connection uses>;\n","sql",[19,337,338,343],{"__ignoreMap":48},[52,339,340],{"class":54,"line":55},[52,341,342],{},"USE ROLE ACCOUNTADMIN;\n",[52,344,345],{"class":54,"line":61},[52,346,347],{},"GRANT EXECUTE TASK ON ACCOUNT TO ROLE \u003Cthe role your connection uses>;\n",[349,350],"br",{},"Without it, changes still reach ",[19,353,189],{}," and never reach ",[19,356,161],{},".\nThe stream says so, with that statement in the error, rather than leaving you\nto find an empty table.",[274,359,361],{"id":360},"how-often-changes-are-applied-and-what-it-costs","How often changes are applied, and what it costs",[15,363,364,365,367,368,370,371,374,375,378],{},"Changes reach ",[19,366,189],{}," within seconds, always. What you choose is how\noften they are merged into ",[19,369,161],{}," — the schedule of the apply task, set in the\nstream wizard under ",[24,372,373],{},"Applying changes"," and as\n",[19,376,377],{},"target.spec.snowflake.currentState"," through the API.",[380,381],"themed-image",{"alt":382,"dark":383,"light":384},"Structure and Data step of the stream wizard for a Snowflake CDC target: schema policy on the left, and on the right the Applying changes slider set to Applied every five minutes, marked Paid by the warehouse, over a scale running from 1 min to 1 hour","\u002Fimages\u002Fdocs\u002Fconnections\u002Fsnowflake\u002Fapply-schedule-dark.png","\u002Fimages\u002Fdocs\u002Fconnections\u002Fsnowflake\u002Fapply-schedule-light.png",[15,386,387,388,391,392,395,396,399,400,403,404],{},"The slider runs from ",[24,389,390],{},"1 min"," through ",[24,393,394],{},"5 min"," and ",[24,397,398],{},"15 min"," to ",[24,401,402],{},"1 hour",",\nand the line above it names the position while the line below says what it costs\n— at five minutes, ",[405,406,407],"em",{},"At most twelve wake-ups an hour. A fair balance, and the\ndefault.",[15,409,410,411,200,414,417],{},"The slider offers four intervals; the API takes any of them and everything in\nbetween, written as a number and a unit — ",[19,412,413],{},"5 minutes",[19,415,416],{},"2 hours"," — from one\nminute to eight days, which is the range a Snowflake task allows. Anything\noutside it is refused before the stream starts rather than part-way through\ncreating the objects.",[15,419,420,421,424],{},"The cost is not the merge itself; it is waking the warehouse. Snowflake bills a\n",[24,422,423],{},"minimum of one minute"," of warehouse time each time one resumes, so what the\nschedule really sets is how many of those you buy an hour:",[164,426,427,440],{},[167,428,429],{},[170,430,431,434,437],{},[173,432,433],{},"Schedule",[173,435,436],{},"Warehouse wake-ups, at most",[173,438,439],{},"Suits",[180,441,442,455,467],{},[170,443,444,449,452],{},[185,445,446],{},[19,447,448],{},"1 minute",[185,450,451],{},"60 an hour — enough that a warehouse may never suspend",[185,453,454],{},"A table read constantly, where minutes matter",[170,456,457,461,464],{},[185,458,459],{},[19,460,413],{},[185,462,463],{},"12 an hour",[185,465,466],{},"The default, and a fair balance",[170,468,469,473,476],{},[185,470,471],{},[19,472,402],{},[185,474,475],{},"1 an hour",[185,477,478],{},"A table read a few times a day",[15,480,481,482,485],{},"A run that finds nothing checks ",[19,483,484],{},"SYSTEM$STREAM_HAS_DATA"," and does not wake the\nwarehouse at all, so a quiet stream costs nothing whatever the schedule says.",[274,487,489],{"id":488},"related-docs","Related docs",[307,491,492,497,503,509],{},[310,493,494,496],{},[32,495,35],{"href":34}," — connection, authentication and Load mode",[310,498,499],{},[32,500,502],{"href":501},"\u002Fdocs\u002Fintegration\u002Fwhat-is-cdc","CDC Mode",[310,504,505],{},[32,506,508],{"href":507},"\u002Fdocs\u002Fstreams\u002Ftarget-data-policies","Target Schema and Data Policies",[310,510,511],{},[32,512,514],{"href":513},"\u002Fdocs\u002Fstreams\u002Fstream-config-reference","Stream Configuration Reference",[516,517,518],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":48,"searchDepth":61,"depth":61,"links":520},[521,522,523,524],{"id":276,"depth":61,"text":277},{"id":304,"depth":61,"text":305},{"id":360,"depth":61,"text":361},{"id":488,"depth":61,"text":489},"Change data capture from MySQL and PostgreSQL into Snowflake with DBConvert Streams — Snowpipe Streaming, the change table, and the task that merges changes into the table you query.","md",{},false,"\u002Fdocs\u002Fconnections\u002Fsnowflake-cdc",null,{"title":5,"description":525},"docs\u002Fconnections\u002Fsnowflake-cdc","9tRMMbgyF19GNH7xO4R6AOoDg8JtawboyFbxC5ZyHdI",1789058293762]